From 607c9eccbffdd1c17f437cb236f7c63d176df4c5 Mon Sep 17 00:00:00 2001 From: Dave Page Date: Wed, 14 Oct 2026 18:20:56 +0100 Subject: [PATCH] Remove unused Google API documents, Babel locale data and packaging tools from the packages. #9331 (#10576) --- Dockerfile | 7 + Make.bat | 5 + pkg/linux/build-functions.sh | 5 + pkg/mac/build-functions.sh | 5 + pkg/prune_site_packages.py | 215 ++++++++++++++++++ web/config.py | 3 + web/pgacloud/providers/google.py | 2 + web/pgadmin/misc/cloud/google/__init__.py | 3 +- .../utils/tests/test_prune_site_packages.py | 184 +++++++++++++++ 9 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 pkg/prune_site_packages.py create mode 100644 web/pgadmin/utils/tests/test_prune_site_packages.py diff --git a/Dockerfile b/Dockerfile index d4e2cb086..05f15c55e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,6 +77,13 @@ RUN python3 -m venv --system-site-packages --without-pip /venv && \ /venv/bin/python3 -m pip install --no-cache-dir -r requirements.txt && \ apk del --no-cache build-deps +# Remove the files pgAdmin never uses. The script reads the language list from +# web/config.py, so it is copied in separately to avoid a change to it +# reinstalling the requirements above. +COPY pkg/prune_site_packages.py /prune/pkg/ +COPY web/config.py /prune/web/ +RUN /venv/bin/python3 /prune/pkg/prune_site_packages.py /venv + ######################################################################### # Now, create a documentation build container for the Sphinx docs ######################################################################### diff --git a/Make.bat b/Make.bat index 19f187723..49eb46490 100644 --- a/Make.bat +++ b/Make.bat @@ -237,6 +237,11 @@ REM Main build sequence Ends ECHO Copying site-packages... XCOPY /S /I /E /H /Y "%TMPDIR%\venv\Lib\site-packages" "%BUILDROOT%\python\Lib\site-packages" > nul || EXIT /B 1 + REM This is done to the copy rather than the venv, which still needs pip to + REM install Sphinx for the docs in :CREATE_RUNTIME_ENV. + ECHO Removing unused files from the installed packages... + "%TMPDIR%\venv\Scripts\python.exe" "%WD%\pkg\prune_site_packages.py" "%BUILDROOT%\python\Lib\site-packages" || EXIT /B 1 + REM NOTE: There is intentionally no space after "site" in the line below, to prevent Python barfing if there's one in the file ECHO import site>> "%BUILDROOT%\python\python%PYTHON_MAJOR%%PYTHON_MINOR%._pth" diff --git a/pkg/linux/build-functions.sh b/pkg/linux/build-functions.sh index 3d24f064f..74d5c53c6 100644 --- a/pkg/linux/build-functions.sh +++ b/pkg/linux/build-functions.sh @@ -147,6 +147,11 @@ _create_python_virtualenv() { --no-binary psycopg -r "${SOURCEDIR}/requirements.txt" fi + # Remove the files in the installed packages that we never use, and the + # packaging tools (pip, setuptools and wheel) that are only needed to + # build the environment + python3 "${SOURCEDIR}/pkg/prune_site_packages.py" venv || exit 1 + # Fixup the paths in the venv activation scripts sed -i 's/VIRTUAL_ENV=.*/VIRTUAL_ENV="\/usr\/pgadmin4\/venv"/g' venv/bin/activate sed -i 's/setenv VIRTUAL_ENV .*/setenv VIRTUAL_ENV "\/usr\/pgadmin4\/venv"/g' venv/bin/activate.csh diff --git a/pkg/mac/build-functions.sh b/pkg/mac/build-functions.sh index db199c444..1cb407fea 100644 --- a/pkg/mac/build-functions.sh +++ b/pkg/mac/build-functions.sh @@ -123,6 +123,11 @@ _create_python_env() { "${BUNDLE_DIR}/Contents/Frameworks/Python.framework/Versions/Current/bin/python3" -m ensurepip --upgrade || exit 1 "${BUNDLE_DIR}/Contents/Frameworks/Python.framework/Versions/Current/bin/pip3" install --no-cache-dir -r "${SOURCE_DIR}/requirements.txt" || exit 1 + # Remove the files in the installed packages that we never use, and the + # packaging tools (pip, setuptools and wheel) that are only needed to + # build the environment + "${BUNDLE_DIR}/Contents/Frameworks/Python.framework/Versions/Current/bin/python3" "${SOURCE_DIR}/pkg/prune_site_packages.py" "${BUNDLE_DIR}/Contents/Frameworks/Python.framework" || exit 1 + # Make sure all the .so's in the Python env have the executable bit set # so they get properly signed later OLD_IFS=${IFS} diff --git a/pkg/prune_site_packages.py b/pkg/prune_site_packages.py new file mode 100644 index 000000000..267c41492 --- /dev/null +++ b/pkg/prune_site_packages.py @@ -0,0 +1,215 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2026, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +"""Remove files that pgAdmin never uses from an installed Python +environment, to reduce the size of the packages. + +Usage: python3 prune_site_packages.py + +The directory is searched recursively, so it may be a venv, a Python +framework or a site-packages directory. The build fails if a file that is +kept cannot be found, rather than shipping a package that cannot work. +""" + +import ast +import fnmatch +import os +import re +import shutil +import sys + +CONFIG_PY = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', + 'web', 'config.py') + + +def languages(): + """Return the language codes of the translations pgAdmin has, from the + keys of LANGUAGES in web/config.py, without the territory or script + (e.g. 'zh' for 'zh_Hans_CN'). The file is parsed rather than imported, + as importing it needs pgAdmin's own modules.""" + with open(CONFIG_PY, encoding='utf-8') as f: + tree = ast.parse(f.read(), CONFIG_PY) + + for node in tree.body: + if isinstance(node, ast.Assign) and \ + any(isinstance(t, ast.Name) and t.id == 'LANGUAGES' + for t in node.targets): + return sorted({code.split('_')[0] + for code in ast.literal_eval(node.value)}) + + sys.exit('ERROR: LANGUAGES was not found in {}'.format(CONFIG_PY)) + + +# Each entry maps a directory to the files in it that may be removed, and +# the patterns for the ones to keep. Every keep pattern must match at least +# one file. +PRUNE = { + # google-api-python-client bundles the discovery document of every + # Google API (600 or so files, over 100MB), and discovery.build() reads + # them from there rather than fetching them. pgAdmin only builds the + # services below; see the *_api_version attributes in + # web/pgadmin/misc/cloud/google/__init__.py and the discovery.build() + # calls in web/pgacloud/providers/google.py, which must be kept in step + # with this list. + os.path.join('googleapiclient', 'discovery_cache', 'documents'): ( + '*.json', [ + 'cloudresourcemanager.v1.json', + 'compute.v1.json', + 'sqladmin.v1.json', + 'sqladmin.v1beta4.json', + ]), + + # Babel carries CLDR data for over 1,000 locales (about 29MB), and only + # loads the user's locale and its parents, down to root. Keeping every + # territory and script variant of each language covers the parents. + os.path.join('babel', 'locale-data'): ( + '*.dat', + ['root.dat'] + + ['{}{}.dat'.format(lang, suffix) + for lang in languages() for suffix in ('', '_*')]), +} + +# Packaging tools that are needed to build the environment but not to run +# pgAdmin. Each is removed from any site-packages directory it is found in. +DISTRIBUTIONS = ['pip', 'setuptools', 'wheel'] + +# A module-level import of pkg_resources, which setuptools provides. Imports +# inside a function or a try block are indented, and are either lazy or +# guarded against it being missing. +PKG_RESOURCES_IMPORT = re.compile(r'^(?:import|from)[ \t]+pkg_resources\b', + re.MULTILINE) + + +def pkg_resources_users(site_packages): + """Return the modules in site_packages, other than the packaging tools' + own and tests, that import pkg_resources when they are loaded. On Python + 3.9, passlib 1.7.4 does, so setuptools must be kept there (#9829).""" + own = {'pip', 'setuptools', 'pkg_resources', '_distutils_hack', 'wheel'} + users = [] + + for dirpath, dirnames, filenames in os.walk(site_packages): + if dirpath == site_packages: + dirnames[:] = [d for d in dirnames if d not in own] + dirnames[:] = [d for d in dirnames if d not in ('test', 'tests')] + for filename in fnmatch.filter(filenames, '*.py'): + path = os.path.join(dirpath, filename) + with open(path, encoding='utf-8', errors='replace') as f: + if PKG_RESOURCES_IMPORT.search(f.read()): + users.append(os.path.relpath(path, site_packages)) + + return sorted(users) + + +def prune_dir(dirpath, removable, keep): + """Remove the files in dirpath matching removable but none of the keep + patterns, and return the number of bytes freed.""" + filenames = os.listdir(dirpath) + + missing = [pattern for pattern in keep + if not fnmatch.filter(filenames, pattern)] + if missing: + sys.exit('ERROR: {} is missing from {}'.format( + ', '.join(missing), dirpath)) + + saved = 0 + for filename in fnmatch.filter(filenames, removable): + if any(fnmatch.fnmatch(filename, pattern) for pattern in keep): + continue + path = os.path.join(dirpath, filename) + saved += os.path.getsize(path) + os.remove(path) + + return saved + + +def _tree_size(path): + return sum(os.path.getsize(os.path.join(dirpath, filename)) + for dirpath, _, filenames in os.walk(path) + for filename in filenames) + + +def uninstall(site_packages, dist_info, root): + """Remove the files listed in a distribution's RECORD, as pip uninstall + would, and return the number of bytes freed. Files outside root, such as + the scripts in a Windows venv when only site-packages is being pruned, + are left alone.""" + root = os.path.abspath(root) + saved = 0 + packages = set() + + # Read the whole of RECORD first, as it lists itself, and Windows cannot + # remove a file that is still open. + with open(os.path.join(site_packages, dist_info, 'RECORD'), + encoding='utf-8') as f: + lines = f.readlines() + + for line in lines: + relpath = line.rsplit(',', 2)[0] + if not relpath: + continue + path = os.path.abspath(os.path.join(site_packages, relpath)) + if os.path.commonpath([root, path]) != root: + continue + if os.sep in os.path.normpath(relpath) and \ + not relpath.startswith('..'): + packages.add(os.path.normpath(relpath).split(os.sep)[0]) + if os.path.isfile(path) or os.path.islink(path): + saved += os.path.getsize(path) + os.remove(path) + + # Remove the package directories too, including any bytecode compiled + # after installation, which RECORD does not list. + for package in packages: + path = os.path.join(site_packages, package) + if os.path.isdir(path): + saved += _tree_size(path) + shutil.rmtree(path) + + return saved + + +def prune(root): + found = set() + saved = 0 + removed = [] + + for dirpath, dirnames, _ in os.walk(root): + for subdir, (removable, keep) in PRUNE.items(): + if dirpath.endswith(os.sep + subdir): + found.add(subdir) + saved += prune_dir(dirpath, removable, keep) + + if os.path.basename(dirpath) == 'site-packages': + users = pkg_resources_users(dirpath) + for name in DISTRIBUTIONS: + if name == 'setuptools' and users: + print('Keeping setuptools, as pkg_resources is imported ' + 'by {}'.format(', '.join(users))) + continue + for dist_info in fnmatch.filter(dirnames, + name + '-*.dist-info'): + saved += uninstall(dirpath, dist_info, root) + removed.append(dist_info[:-len('.dist-info')]) + dirnames[:] = [d for d in dirnames + if os.path.isdir(os.path.join(dirpath, d))] + + for subdir in set(PRUNE) - found: + sys.exit('ERROR: {} was not found under {}'.format(subdir, root)) + + if removed: + print('Removed {}'.format(', '.join(sorted(removed)))) + print('Pruned {:.1f}MB of unused files from {}'.format( + saved / 1024 / 1024, root)) + + +if __name__ == '__main__': + if len(sys.argv) != 2 or not os.path.isdir(sys.argv[1]): + sys.exit('Usage: {} '.format(sys.argv[0])) + + prune(sys.argv[1]) diff --git a/web/config.py b/web/config.py index 9d9f79fbb..eac09381b 100644 --- a/web/config.py +++ b/web/config.py @@ -44,6 +44,9 @@ from branding import APP_NAME, APP_ICON, APP_COPYRIGHT, APP_PATH, \ HELP_PATH = '../../../docs/en_US/_build/html/' # Languages we support in the UI +# Babel's locale data for other languages is removed from the packages by +# pkg/prune_site_packages.py, which parses this assignment, so it must stay a +# plain dict literal. LANGUAGES = { 'en': 'English', 'zh_Hans_CN': 'Chinese (Simplified)', diff --git a/web/pgacloud/providers/google.py b/web/pgacloud/providers/google.py index 5d34b50c8..d4f12b8c3 100644 --- a/web/pgacloud/providers/google.py +++ b/web/pgacloud/providers/google.py @@ -145,6 +145,8 @@ class GoogleProvider(AbsProvider): def _create_google_postgresql_instance(self, args): credentials = self._get_credentials(self._scopes) + # Only the discovery documents for the services pgAdmin builds are + # shipped in the packages; see pkg/prune_site_packages.py. service = discovery.build('sqladmin', 'v1beta4', credentials=credentials) diff --git a/web/pgadmin/misc/cloud/google/__init__.py b/web/pgadmin/misc/cloud/google/__init__.py index 31e3006f7..73fe2e1e8 100644 --- a/web/pgadmin/misc/cloud/google/__init__.py +++ b/web/pgadmin/misc/cloud/google/__init__.py @@ -388,7 +388,8 @@ def _google_sdk(): class Google: def __init__(self, client_config=None): - # Google cloud sql api versions + # Google cloud sql api versions. Only the discovery documents for + # these are shipped in the packages; see pkg/prune_site_packages.py. self._cloud_resource_manager_api_version = 'v1' self._sqladmin_api_version = 'v1' self._compute_api_version = 'v1' diff --git a/web/pgadmin/utils/tests/test_prune_site_packages.py b/web/pgadmin/utils/tests/test_prune_site_packages.py new file mode 100644 index 000000000..eb1c57213 --- /dev/null +++ b/web/pgadmin/utils/tests/test_prune_site_packages.py @@ -0,0 +1,184 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2026, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +""" +The packages run pkg/prune_site_packages.py to remove files from the +installed Python packages that pgAdmin never uses. Prune a copy of each +affected directory, point the library at it, and check that everything +pgAdmin needs from it still works. +""" + +import ast +import importlib.util +import os +import shutil +import tempfile +import unittest +from datetime import datetime +from unittest import mock + +import config +from pgadmin.utils.route import BaseTestGenerator + +PRUNE_SCRIPT = os.path.abspath(os.path.join( + os.path.dirname(__file__), *(['..'] * 4), 'pkg', + 'prune_site_packages.py')) +PGACLOUD_GOOGLE = os.path.abspath(os.path.join( + os.path.dirname(__file__), *(['..'] * 3), 'pgacloud', 'providers', + 'google.py')) + + +def _pgacloud_services(): + """Return the (service, version) pairs that pgacloud's Google provider + passes to discovery.build().""" + with open(PGACLOUD_GOOGLE, encoding='utf-8') as f: + tree = ast.parse(f.read(), PGACLOUD_GOOGLE) + return {tuple(arg.value for arg in node.args[:2]) + for node in ast.walk(tree) + if isinstance(node, ast.Call) and + isinstance(node.func, ast.Attribute) and + node.func.attr == 'build' and len(node.args) >= 2 and + all(isinstance(arg, ast.Constant) for arg in node.args[:2])} + + +class _SkipServerSetUpMixin: + """Skip BaseTestGenerator's Postgres connection — pure logic tests.""" + + def setUp(self): + unittest.TestCase.setUp(self) + + +class TestPruneSitePackages(_SkipServerSetUpMixin, BaseTestGenerator): + """What pgAdmin uses survives pruning the installed packages.""" + + scenarios = ( + ('Google discovery documents', dict(check='google')), + ('Babel locale data', dict(check='babel')), + ('Packaging tools', dict(check='packaging')), + ) + + def runTest(self): + if not os.path.isfile(PRUNE_SCRIPT): + self.skipTest('{} not found'.format(PRUNE_SCRIPT)) + + spec = importlib.util.spec_from_file_location( + 'prune_site_packages', PRUNE_SCRIPT) + self.prune = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.prune) + + getattr(self, '_check_' + self.check)() + + def _pruned_copy(self, source, subdir): + """Copy source, prune it as subdir would be and return the copy.""" + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + target = os.path.join(tmp, 'data') + shutil.copytree(source, target) + self.prune.prune_dir(target, *self.prune.PRUNE[subdir]) + return target + + def _check_google(self): + from googleapiclient import discovery_cache + from pgadmin.misc.cloud.google import Google, _google_sdk + + docs = self._pruned_copy( + discovery_cache.DISCOVERY_DOC_DIR, + os.path.join('googleapiclient', 'discovery_cache', 'documents')) + + google = Google() + services = { + ('cloudresourcemanager', + google._cloud_resource_manager_api_version), + ('compute', google._compute_api_version), + ('sqladmin', google._sqladmin_api_version), + } + pgacloud_services = _pgacloud_services() + self.assertTrue(pgacloud_services) + services |= pgacloud_services + + discovery = _google_sdk().discovery + with mock.patch.object(discovery_cache, 'DISCOVERY_DOC_DIR', docs): + for service, version in sorted(services): + with self.subTest(service=service, version=version): + # Builds offline from the bundled document. + discovery.build(service, version, developerKey='test', + static_discovery=True) + + def _check_babel(self): + from babel import Locale, localedata + from babel.dates import format_datetime + + data = self._pruned_copy( + localedata._dirname, os.path.join('babel', 'locale-data')) + + localedata.locale_identifiers.cache_clear() + self.addCleanup(localedata.locale_identifiers.cache_clear) + with mock.patch.object(localedata, '_dirname', data), \ + mock.patch.dict(localedata._cache, clear=True): + for language in config.LANGUAGES: + with self.subTest(language=language): + locale = Locale.parse(language) + self.assertTrue(locale.display_name) + self.assertTrue(format_datetime(datetime(2026, 1, 2), + locale=locale)) + + def _check_packaging(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + site_packages = os.path.join(tmp, 'lib', 'site-packages') + files = { + os.path.join('pip', '__init__.py'): 'pip/__init__.py', + os.path.join('pip', '__pycache__', 'x.pyc'): None, + os.path.join('pip-1.0.dist-info', 'RECORD'): + 'pip-1.0.dist-info/RECORD', + os.path.join('..', 'bin', 'pip'): '../bin/pip', + os.path.join('keep', '__init__.py'): None, + } + for path, record in files.items(): + path = os.path.join(site_packages, path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write('x') + with open(os.path.join(site_packages, 'pip-1.0.dist-info', + 'RECORD'), 'w') as f: + f.write(''.join('{},,\n'.format(record) + for record in files.values() if record)) + + # Pruning only site-packages leaves the scripts outside it alone. + self.prune.uninstall(site_packages, 'pip-1.0.dist-info', + site_packages) + self.assertEqual(sorted(os.listdir(site_packages)), ['keep']) + self.assertTrue(os.path.exists(os.path.join(tmp, 'lib', 'bin', + 'pip'))) + + # setuptools is kept if anything imports pkg_resources when it is + # loaded, as passlib 1.7.4 does on Python 3.9 (#9829), but not for + # lazy or guarded imports, or tests. + modules = { + os.path.join('keep', '__init__.py'): None, + os.path.join('lazy', '__init__.py'): + 'def f():\n import pkg_resources\n', + os.path.join('keep', 'tests', 'test_x.py'): + 'import pkg_resources\n', + os.path.join('pkg_resources', '__init__.py'): + 'from pkg_resources.extern import x\n', + } + for path, content in modules.items(): + path = os.path.join(site_packages, path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write(content or '') + self.assertEqual(self.prune.pkg_resources_users(site_packages), []) + + path = os.path.join(site_packages, 'passlib', 'pwd.py') + os.makedirs(os.path.dirname(path)) + with open(path, 'w') as f: + f.write('import os\nimport pkg_resources\n') + self.assertEqual(self.prune.pkg_resources_users(site_packages), + [os.path.join('passlib', 'pwd.py')])