Remove unused Google API documents, Babel locale data and packaging tools from the packages. #9331 (#10576)
This commit is contained in:
Dave Page
@@ -77,6 +77,13 @@ RUN python3 -m venv --system-site-packages --without-pip /venv && \
|
||||
/venv/bin/python3 -m pip install --no-cache-dir -r requirements.txt && \
|
||||
apk del --no-cache build-deps
|
||||
|
||||
# Remove the files pgAdmin never uses. The script reads the language list from
|
||||
# web/config.py, so it is copied in separately to avoid a change to it
|
||||
# reinstalling the requirements above.
|
||||
COPY pkg/prune_site_packages.py /prune/pkg/
|
||||
COPY web/config.py /prune/web/
|
||||
RUN /venv/bin/python3 /prune/pkg/prune_site_packages.py /venv
|
||||
|
||||
#########################################################################
|
||||
# Now, create a documentation build container for the Sphinx docs
|
||||
#########################################################################
|
||||
|
||||
5
Make.bat
5
Make.bat
@@ -237,6 +237,11 @@ REM Main build sequence Ends
|
||||
ECHO Copying site-packages...
|
||||
XCOPY /S /I /E /H /Y "%TMPDIR%\venv\Lib\site-packages" "%BUILDROOT%\python\Lib\site-packages" > nul || EXIT /B 1
|
||||
|
||||
REM This is done to the copy rather than the venv, which still needs pip to
|
||||
REM install Sphinx for the docs in :CREATE_RUNTIME_ENV.
|
||||
ECHO Removing unused files from the installed packages...
|
||||
"%TMPDIR%\venv\Scripts\python.exe" "%WD%\pkg\prune_site_packages.py" "%BUILDROOT%\python\Lib\site-packages" || EXIT /B 1
|
||||
|
||||
REM NOTE: There is intentionally no space after "site" in the line below, to prevent Python barfing if there's one in the file
|
||||
ECHO import site>> "%BUILDROOT%\python\python%PYTHON_MAJOR%%PYTHON_MINOR%._pth"
|
||||
|
||||
|
||||
@@ -147,6 +147,11 @@ _create_python_virtualenv() {
|
||||
--no-binary psycopg -r "${SOURCEDIR}/requirements.txt"
|
||||
fi
|
||||
|
||||
# Remove the files in the installed packages that we never use, and the
|
||||
# packaging tools (pip, setuptools and wheel) that are only needed to
|
||||
# build the environment
|
||||
python3 "${SOURCEDIR}/pkg/prune_site_packages.py" venv || exit 1
|
||||
|
||||
# Fixup the paths in the venv activation scripts
|
||||
sed -i 's/VIRTUAL_ENV=.*/VIRTUAL_ENV="\/usr\/pgadmin4\/venv"/g' venv/bin/activate
|
||||
sed -i 's/setenv VIRTUAL_ENV .*/setenv VIRTUAL_ENV "\/usr\/pgadmin4\/venv"/g' venv/bin/activate.csh
|
||||
|
||||
@@ -123,6 +123,11 @@ _create_python_env() {
|
||||
"${BUNDLE_DIR}/Contents/Frameworks/Python.framework/Versions/Current/bin/python3" -m ensurepip --upgrade || exit 1
|
||||
"${BUNDLE_DIR}/Contents/Frameworks/Python.framework/Versions/Current/bin/pip3" install --no-cache-dir -r "${SOURCE_DIR}/requirements.txt" || exit 1
|
||||
|
||||
# Remove the files in the installed packages that we never use, and the
|
||||
# packaging tools (pip, setuptools and wheel) that are only needed to
|
||||
# build the environment
|
||||
"${BUNDLE_DIR}/Contents/Frameworks/Python.framework/Versions/Current/bin/python3" "${SOURCE_DIR}/pkg/prune_site_packages.py" "${BUNDLE_DIR}/Contents/Frameworks/Python.framework" || exit 1
|
||||
|
||||
# Make sure all the .so's in the Python env have the executable bit set
|
||||
# so they get properly signed later
|
||||
OLD_IFS=${IFS}
|
||||
|
||||
215
pkg/prune_site_packages.py
Normal file
215
pkg/prune_site_packages.py
Normal file
@@ -0,0 +1,215 @@
|
||||
##########################################################################
|
||||
#
|
||||
# pgAdmin 4 - PostgreSQL Tools
|
||||
#
|
||||
# Copyright (C) 2013 - 2026, The pgAdmin Development Team
|
||||
# This software is released under the PostgreSQL Licence
|
||||
#
|
||||
##########################################################################
|
||||
|
||||
"""Remove files that pgAdmin never uses from an installed Python
|
||||
environment, to reduce the size of the packages.
|
||||
|
||||
Usage: python3 prune_site_packages.py <directory>
|
||||
|
||||
The directory is searched recursively, so it may be a venv, a Python
|
||||
framework or a site-packages directory. The build fails if a file that is
|
||||
kept cannot be found, rather than shipping a package that cannot work.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import fnmatch
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
CONFIG_PY = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
|
||||
'web', 'config.py')
|
||||
|
||||
|
||||
def languages():
|
||||
"""Return the language codes of the translations pgAdmin has, from the
|
||||
keys of LANGUAGES in web/config.py, without the territory or script
|
||||
(e.g. 'zh' for 'zh_Hans_CN'). The file is parsed rather than imported,
|
||||
as importing it needs pgAdmin's own modules."""
|
||||
with open(CONFIG_PY, encoding='utf-8') as f:
|
||||
tree = ast.parse(f.read(), CONFIG_PY)
|
||||
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign) and \
|
||||
any(isinstance(t, ast.Name) and t.id == 'LANGUAGES'
|
||||
for t in node.targets):
|
||||
return sorted({code.split('_')[0]
|
||||
for code in ast.literal_eval(node.value)})
|
||||
|
||||
sys.exit('ERROR: LANGUAGES was not found in {}'.format(CONFIG_PY))
|
||||
|
||||
|
||||
# Each entry maps a directory to the files in it that may be removed, and
|
||||
# the patterns for the ones to keep. Every keep pattern must match at least
|
||||
# one file.
|
||||
PRUNE = {
|
||||
# google-api-python-client bundles the discovery document of every
|
||||
# Google API (600 or so files, over 100MB), and discovery.build() reads
|
||||
# them from there rather than fetching them. pgAdmin only builds the
|
||||
# services below; see the *_api_version attributes in
|
||||
# web/pgadmin/misc/cloud/google/__init__.py and the discovery.build()
|
||||
# calls in web/pgacloud/providers/google.py, which must be kept in step
|
||||
# with this list.
|
||||
os.path.join('googleapiclient', 'discovery_cache', 'documents'): (
|
||||
'*.json', [
|
||||
'cloudresourcemanager.v1.json',
|
||||
'compute.v1.json',
|
||||
'sqladmin.v1.json',
|
||||
'sqladmin.v1beta4.json',
|
||||
]),
|
||||
|
||||
# Babel carries CLDR data for over 1,000 locales (about 29MB), and only
|
||||
# loads the user's locale and its parents, down to root. Keeping every
|
||||
# territory and script variant of each language covers the parents.
|
||||
os.path.join('babel', 'locale-data'): (
|
||||
'*.dat',
|
||||
['root.dat'] +
|
||||
['{}{}.dat'.format(lang, suffix)
|
||||
for lang in languages() for suffix in ('', '_*')]),
|
||||
}
|
||||
|
||||
# Packaging tools that are needed to build the environment but not to run
|
||||
# pgAdmin. Each is removed from any site-packages directory it is found in.
|
||||
DISTRIBUTIONS = ['pip', 'setuptools', 'wheel']
|
||||
|
||||
# A module-level import of pkg_resources, which setuptools provides. Imports
|
||||
# inside a function or a try block are indented, and are either lazy or
|
||||
# guarded against it being missing.
|
||||
PKG_RESOURCES_IMPORT = re.compile(r'^(?:import|from)[ \t]+pkg_resources\b',
|
||||
re.MULTILINE)
|
||||
|
||||
|
||||
def pkg_resources_users(site_packages):
|
||||
"""Return the modules in site_packages, other than the packaging tools'
|
||||
own and tests, that import pkg_resources when they are loaded. On Python
|
||||
3.9, passlib 1.7.4 does, so setuptools must be kept there (#9829)."""
|
||||
own = {'pip', 'setuptools', 'pkg_resources', '_distutils_hack', 'wheel'}
|
||||
users = []
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(site_packages):
|
||||
if dirpath == site_packages:
|
||||
dirnames[:] = [d for d in dirnames if d not in own]
|
||||
dirnames[:] = [d for d in dirnames if d not in ('test', 'tests')]
|
||||
for filename in fnmatch.filter(filenames, '*.py'):
|
||||
path = os.path.join(dirpath, filename)
|
||||
with open(path, encoding='utf-8', errors='replace') as f:
|
||||
if PKG_RESOURCES_IMPORT.search(f.read()):
|
||||
users.append(os.path.relpath(path, site_packages))
|
||||
|
||||
return sorted(users)
|
||||
|
||||
|
||||
def prune_dir(dirpath, removable, keep):
|
||||
"""Remove the files in dirpath matching removable but none of the keep
|
||||
patterns, and return the number of bytes freed."""
|
||||
filenames = os.listdir(dirpath)
|
||||
|
||||
missing = [pattern for pattern in keep
|
||||
if not fnmatch.filter(filenames, pattern)]
|
||||
if missing:
|
||||
sys.exit('ERROR: {} is missing from {}'.format(
|
||||
', '.join(missing), dirpath))
|
||||
|
||||
saved = 0
|
||||
for filename in fnmatch.filter(filenames, removable):
|
||||
if any(fnmatch.fnmatch(filename, pattern) for pattern in keep):
|
||||
continue
|
||||
path = os.path.join(dirpath, filename)
|
||||
saved += os.path.getsize(path)
|
||||
os.remove(path)
|
||||
|
||||
return saved
|
||||
|
||||
|
||||
def _tree_size(path):
|
||||
return sum(os.path.getsize(os.path.join(dirpath, filename))
|
||||
for dirpath, _, filenames in os.walk(path)
|
||||
for filename in filenames)
|
||||
|
||||
|
||||
def uninstall(site_packages, dist_info, root):
|
||||
"""Remove the files listed in a distribution's RECORD, as pip uninstall
|
||||
would, and return the number of bytes freed. Files outside root, such as
|
||||
the scripts in a Windows venv when only site-packages is being pruned,
|
||||
are left alone."""
|
||||
root = os.path.abspath(root)
|
||||
saved = 0
|
||||
packages = set()
|
||||
|
||||
# Read the whole of RECORD first, as it lists itself, and Windows cannot
|
||||
# remove a file that is still open.
|
||||
with open(os.path.join(site_packages, dist_info, 'RECORD'),
|
||||
encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
for line in lines:
|
||||
relpath = line.rsplit(',', 2)[0]
|
||||
if not relpath:
|
||||
continue
|
||||
path = os.path.abspath(os.path.join(site_packages, relpath))
|
||||
if os.path.commonpath([root, path]) != root:
|
||||
continue
|
||||
if os.sep in os.path.normpath(relpath) and \
|
||||
not relpath.startswith('..'):
|
||||
packages.add(os.path.normpath(relpath).split(os.sep)[0])
|
||||
if os.path.isfile(path) or os.path.islink(path):
|
||||
saved += os.path.getsize(path)
|
||||
os.remove(path)
|
||||
|
||||
# Remove the package directories too, including any bytecode compiled
|
||||
# after installation, which RECORD does not list.
|
||||
for package in packages:
|
||||
path = os.path.join(site_packages, package)
|
||||
if os.path.isdir(path):
|
||||
saved += _tree_size(path)
|
||||
shutil.rmtree(path)
|
||||
|
||||
return saved
|
||||
|
||||
|
||||
def prune(root):
|
||||
found = set()
|
||||
saved = 0
|
||||
removed = []
|
||||
|
||||
for dirpath, dirnames, _ in os.walk(root):
|
||||
for subdir, (removable, keep) in PRUNE.items():
|
||||
if dirpath.endswith(os.sep + subdir):
|
||||
found.add(subdir)
|
||||
saved += prune_dir(dirpath, removable, keep)
|
||||
|
||||
if os.path.basename(dirpath) == 'site-packages':
|
||||
users = pkg_resources_users(dirpath)
|
||||
for name in DISTRIBUTIONS:
|
||||
if name == 'setuptools' and users:
|
||||
print('Keeping setuptools, as pkg_resources is imported '
|
||||
'by {}'.format(', '.join(users)))
|
||||
continue
|
||||
for dist_info in fnmatch.filter(dirnames,
|
||||
name + '-*.dist-info'):
|
||||
saved += uninstall(dirpath, dist_info, root)
|
||||
removed.append(dist_info[:-len('.dist-info')])
|
||||
dirnames[:] = [d for d in dirnames
|
||||
if os.path.isdir(os.path.join(dirpath, d))]
|
||||
|
||||
for subdir in set(PRUNE) - found:
|
||||
sys.exit('ERROR: {} was not found under {}'.format(subdir, root))
|
||||
|
||||
if removed:
|
||||
print('Removed {}'.format(', '.join(sorted(removed))))
|
||||
print('Pruned {:.1f}MB of unused files from {}'.format(
|
||||
saved / 1024 / 1024, root))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2 or not os.path.isdir(sys.argv[1]):
|
||||
sys.exit('Usage: {} <directory>'.format(sys.argv[0]))
|
||||
|
||||
prune(sys.argv[1])
|
||||
@@ -44,6 +44,9 @@ from branding import APP_NAME, APP_ICON, APP_COPYRIGHT, APP_PATH, \
|
||||
HELP_PATH = '../../../docs/en_US/_build/html/'
|
||||
|
||||
# Languages we support in the UI
|
||||
# Babel's locale data for other languages is removed from the packages by
|
||||
# pkg/prune_site_packages.py, which parses this assignment, so it must stay a
|
||||
# plain dict literal.
|
||||
LANGUAGES = {
|
||||
'en': 'English',
|
||||
'zh_Hans_CN': 'Chinese (Simplified)',
|
||||
|
||||
@@ -145,6 +145,8 @@ class GoogleProvider(AbsProvider):
|
||||
|
||||
def _create_google_postgresql_instance(self, args):
|
||||
credentials = self._get_credentials(self._scopes)
|
||||
# Only the discovery documents for the services pgAdmin builds are
|
||||
# shipped in the packages; see pkg/prune_site_packages.py.
|
||||
service = discovery.build('sqladmin', 'v1beta4',
|
||||
credentials=credentials)
|
||||
|
||||
|
||||
@@ -388,7 +388,8 @@ def _google_sdk():
|
||||
|
||||
class Google:
|
||||
def __init__(self, client_config=None):
|
||||
# Google cloud sql api versions
|
||||
# Google cloud sql api versions. Only the discovery documents for
|
||||
# these are shipped in the packages; see pkg/prune_site_packages.py.
|
||||
self._cloud_resource_manager_api_version = 'v1'
|
||||
self._sqladmin_api_version = 'v1'
|
||||
self._compute_api_version = 'v1'
|
||||
|
||||
184
web/pgadmin/utils/tests/test_prune_site_packages.py
Normal file
184
web/pgadmin/utils/tests/test_prune_site_packages.py
Normal file
@@ -0,0 +1,184 @@
|
||||
##########################################################################
|
||||
#
|
||||
# pgAdmin 4 - PostgreSQL Tools
|
||||
#
|
||||
# Copyright (C) 2013 - 2026, The pgAdmin Development Team
|
||||
# This software is released under the PostgreSQL Licence
|
||||
#
|
||||
##########################################################################
|
||||
|
||||
"""
|
||||
The packages run pkg/prune_site_packages.py to remove files from the
|
||||
installed Python packages that pgAdmin never uses. Prune a copy of each
|
||||
affected directory, point the library at it, and check that everything
|
||||
pgAdmin needs from it still works.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest import mock
|
||||
|
||||
import config
|
||||
from pgadmin.utils.route import BaseTestGenerator
|
||||
|
||||
PRUNE_SCRIPT = os.path.abspath(os.path.join(
|
||||
os.path.dirname(__file__), *(['..'] * 4), 'pkg',
|
||||
'prune_site_packages.py'))
|
||||
PGACLOUD_GOOGLE = os.path.abspath(os.path.join(
|
||||
os.path.dirname(__file__), *(['..'] * 3), 'pgacloud', 'providers',
|
||||
'google.py'))
|
||||
|
||||
|
||||
def _pgacloud_services():
|
||||
"""Return the (service, version) pairs that pgacloud's Google provider
|
||||
passes to discovery.build()."""
|
||||
with open(PGACLOUD_GOOGLE, encoding='utf-8') as f:
|
||||
tree = ast.parse(f.read(), PGACLOUD_GOOGLE)
|
||||
return {tuple(arg.value for arg in node.args[:2])
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call) and
|
||||
isinstance(node.func, ast.Attribute) and
|
||||
node.func.attr == 'build' and len(node.args) >= 2 and
|
||||
all(isinstance(arg, ast.Constant) for arg in node.args[:2])}
|
||||
|
||||
|
||||
class _SkipServerSetUpMixin:
|
||||
"""Skip BaseTestGenerator's Postgres connection — pure logic tests."""
|
||||
|
||||
def setUp(self):
|
||||
unittest.TestCase.setUp(self)
|
||||
|
||||
|
||||
class TestPruneSitePackages(_SkipServerSetUpMixin, BaseTestGenerator):
|
||||
"""What pgAdmin uses survives pruning the installed packages."""
|
||||
|
||||
scenarios = (
|
||||
('Google discovery documents', dict(check='google')),
|
||||
('Babel locale data', dict(check='babel')),
|
||||
('Packaging tools', dict(check='packaging')),
|
||||
)
|
||||
|
||||
def runTest(self):
|
||||
if not os.path.isfile(PRUNE_SCRIPT):
|
||||
self.skipTest('{} not found'.format(PRUNE_SCRIPT))
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
'prune_site_packages', PRUNE_SCRIPT)
|
||||
self.prune = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(self.prune)
|
||||
|
||||
getattr(self, '_check_' + self.check)()
|
||||
|
||||
def _pruned_copy(self, source, subdir):
|
||||
"""Copy source, prune it as subdir would be and return the copy."""
|
||||
tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, tmp)
|
||||
target = os.path.join(tmp, 'data')
|
||||
shutil.copytree(source, target)
|
||||
self.prune.prune_dir(target, *self.prune.PRUNE[subdir])
|
||||
return target
|
||||
|
||||
def _check_google(self):
|
||||
from googleapiclient import discovery_cache
|
||||
from pgadmin.misc.cloud.google import Google, _google_sdk
|
||||
|
||||
docs = self._pruned_copy(
|
||||
discovery_cache.DISCOVERY_DOC_DIR,
|
||||
os.path.join('googleapiclient', 'discovery_cache', 'documents'))
|
||||
|
||||
google = Google()
|
||||
services = {
|
||||
('cloudresourcemanager',
|
||||
google._cloud_resource_manager_api_version),
|
||||
('compute', google._compute_api_version),
|
||||
('sqladmin', google._sqladmin_api_version),
|
||||
}
|
||||
pgacloud_services = _pgacloud_services()
|
||||
self.assertTrue(pgacloud_services)
|
||||
services |= pgacloud_services
|
||||
|
||||
discovery = _google_sdk().discovery
|
||||
with mock.patch.object(discovery_cache, 'DISCOVERY_DOC_DIR', docs):
|
||||
for service, version in sorted(services):
|
||||
with self.subTest(service=service, version=version):
|
||||
# Builds offline from the bundled document.
|
||||
discovery.build(service, version, developerKey='test',
|
||||
static_discovery=True)
|
||||
|
||||
def _check_babel(self):
|
||||
from babel import Locale, localedata
|
||||
from babel.dates import format_datetime
|
||||
|
||||
data = self._pruned_copy(
|
||||
localedata._dirname, os.path.join('babel', 'locale-data'))
|
||||
|
||||
localedata.locale_identifiers.cache_clear()
|
||||
self.addCleanup(localedata.locale_identifiers.cache_clear)
|
||||
with mock.patch.object(localedata, '_dirname', data), \
|
||||
mock.patch.dict(localedata._cache, clear=True):
|
||||
for language in config.LANGUAGES:
|
||||
with self.subTest(language=language):
|
||||
locale = Locale.parse(language)
|
||||
self.assertTrue(locale.display_name)
|
||||
self.assertTrue(format_datetime(datetime(2026, 1, 2),
|
||||
locale=locale))
|
||||
|
||||
def _check_packaging(self):
|
||||
tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, tmp)
|
||||
site_packages = os.path.join(tmp, 'lib', 'site-packages')
|
||||
files = {
|
||||
os.path.join('pip', '__init__.py'): 'pip/__init__.py',
|
||||
os.path.join('pip', '__pycache__', 'x.pyc'): None,
|
||||
os.path.join('pip-1.0.dist-info', 'RECORD'):
|
||||
'pip-1.0.dist-info/RECORD',
|
||||
os.path.join('..', 'bin', 'pip'): '../bin/pip',
|
||||
os.path.join('keep', '__init__.py'): None,
|
||||
}
|
||||
for path, record in files.items():
|
||||
path = os.path.join(site_packages, path)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
f.write('x')
|
||||
with open(os.path.join(site_packages, 'pip-1.0.dist-info',
|
||||
'RECORD'), 'w') as f:
|
||||
f.write(''.join('{},,\n'.format(record)
|
||||
for record in files.values() if record))
|
||||
|
||||
# Pruning only site-packages leaves the scripts outside it alone.
|
||||
self.prune.uninstall(site_packages, 'pip-1.0.dist-info',
|
||||
site_packages)
|
||||
self.assertEqual(sorted(os.listdir(site_packages)), ['keep'])
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp, 'lib', 'bin',
|
||||
'pip')))
|
||||
|
||||
# setuptools is kept if anything imports pkg_resources when it is
|
||||
# loaded, as passlib 1.7.4 does on Python 3.9 (#9829), but not for
|
||||
# lazy or guarded imports, or tests.
|
||||
modules = {
|
||||
os.path.join('keep', '__init__.py'): None,
|
||||
os.path.join('lazy', '__init__.py'):
|
||||
'def f():\n import pkg_resources\n',
|
||||
os.path.join('keep', 'tests', 'test_x.py'):
|
||||
'import pkg_resources\n',
|
||||
os.path.join('pkg_resources', '__init__.py'):
|
||||
'from pkg_resources.extern import x\n',
|
||||
}
|
||||
for path, content in modules.items():
|
||||
path = os.path.join(site_packages, path)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
f.write(content or '')
|
||||
self.assertEqual(self.prune.pkg_resources_users(site_packages), [])
|
||||
|
||||
path = os.path.join(site_packages, 'passlib', 'pwd.py')
|
||||
os.makedirs(os.path.dirname(path))
|
||||
with open(path, 'w') as f:
|
||||
f.write('import os\nimport pkg_resources\n')
|
||||
self.assertEqual(self.prune.pkg_resources_users(site_packages),
|
||||
[os.path.join('passlib', 'pwd.py')])
|
||||
Reference in New Issue
Block a user