84a76baa8e
The lock was written against the rolling tags because the per-version releases did not exist when it was generated, and the refresh tool assumed PostgreSQL was tagged by major, which it is not: the release that never changes is postgresql-18.6, whilst only the rolling one says 18. The tool now resolves a major to the version pgbuild's manifest names, and a tag that does not exist is an error rather than something to fall back from, since a lock recording an archive nobody can fetch is worse than no lock at all.
417 lines
15 KiB
Python
Executable File
417 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
##########################################################################
|
|
#
|
|
# pgAdmin 4 - PostgreSQL Tools
|
|
#
|
|
# Copyright (C) 2013 - 2026, The pgAdmin Development Team
|
|
# This software is released under the PostgreSQL Licence
|
|
#
|
|
##########################################################################
|
|
"""Refresh pkg/pgbuild-deps.lock from the archives pgbuild publishes.
|
|
|
|
The install-pgbuild-deps action downloads the build dependencies from the
|
|
release tags this file records, and refuses to unpack anything whose
|
|
SHA-256 is not the one recorded alongside, so bumping a dependency means
|
|
running this, looking at the diff and committing it. That diff is the audit
|
|
trail: it is the only place a change in what pgAdmin signs and ships is
|
|
written down and reviewed by a person.
|
|
|
|
pgbuild tags an immutable release per upstream version, in the shape
|
|
<package>-<version>-<platform>-<arch>, and the version it built is recorded
|
|
in its manifest.json, which is where the tag to pin is worked out from. The
|
|
archive is then downloaded from that tag and hashed. pgbuild publishes a
|
|
.sha256 beside each archive, which this deliberately ignores: it is
|
|
regenerated by whoever regenerates the archive, so it is a convenience
|
|
rather than a source of trust, and the lock file here is the only thing
|
|
that pins anything.
|
|
|
|
Refreshing everything restates every digest, which buries a single genuine
|
|
bump in noise, so name the packages you meant to bump:
|
|
|
|
tools/refresh_pgbuild_lock.py openssl
|
|
tools/refresh_pgbuild_lock.py postgresql krb5
|
|
tools/refresh_pgbuild_lock.py # everything, on purpose
|
|
|
|
Entries for packages you do not name are carried through from the existing
|
|
lock file untouched. To pin something other than what pgbuild's manifest
|
|
currently names, give the tag outright:
|
|
|
|
tools/refresh_pgbuild_lock.py \
|
|
--tag openssl-macos-arm64=openssl-3.5.3-macos-arm64
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import os
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
REPO = 'pgadmin-org/pgbuild'
|
|
BASE_URL = 'https://github.com/%s/releases/download' % REPO
|
|
MANIFEST_URL = 'https://raw.githubusercontent.com/%s/HEAD/manifest.json' \
|
|
% REPO
|
|
|
|
# What install-pgbuild-deps fetches on each platform. These lists have to
|
|
# track the action, because a package the action downloads but which is
|
|
# missing from here would have no lock entry and would fail every build.
|
|
MACOS_PACKAGES = ['openssl', 'krb5', 'zstd', 'lz4', 'postgresql']
|
|
MACOS_ARCHES = ['arm64', 'x86_64']
|
|
WINDOWS_PACKAGES = ['postgresql', 'krb5']
|
|
WINDOWS_ARCHES = ['x86_64']
|
|
|
|
# The PostgreSQL majors the workflows ask the action for. Only 18 is used
|
|
# today, both as the action's default and as the explicit value the Windows
|
|
# workflows pass, but a major is added here the moment a workflow names it,
|
|
# since an archive with no lock entry does not build.
|
|
POSTGRES_MAJORS = ['18']
|
|
|
|
HEADER = """\
|
|
# pgbuild dependency lock
|
|
#
|
|
# Which build of each dependency .github/actions/install-pgbuild-deps
|
|
# downloads from pgadmin-org/pgbuild, and what it must hash to. The action
|
|
# fetches the tag named here and refuses to unpack an archive whose SHA-256
|
|
# differs, so these lines decide which PostgreSQL, OpenSSL and Kerberos
|
|
# binaries end up inside an Authenticode signed installer and a notarised
|
|
# appbundle.
|
|
#
|
|
# It matters because pgbuild also publishes rolling tags ending in -latest,
|
|
# whose assets are replaced in place on every rebuild. Building from those
|
|
# means signing whatever they happened to point at that morning, write
|
|
# access to pgbuild is write access to what pgAdmin ships, and no two
|
|
# builds of the same commit contain the same binaries. Pinning the
|
|
# per-version tag instead gives a build that can be reproduced and a change
|
|
# that can be reviewed. pgbuild publishes a .sha256 beside each archive,
|
|
# which is a convenience for people rather than a source of trust, since
|
|
# whoever rebuilds the archive rebuilds that too; this file is the only
|
|
# thing that actually pins anything, which is why the action checks against
|
|
# it and not against the published checksum.
|
|
#
|
|
# Three whitespace separated columns, one dependency per line, with # for a
|
|
# comment. Kept dull on purpose so that bash, PowerShell and a reviewer can
|
|
# all read it.
|
|
#
|
|
# 1. the dependency, as the action names it: package, platform and
|
|
# architecture, with the PostgreSQL major where there is one
|
|
# 2. the pgbuild release tag to download from, which carries the upstream
|
|
# version, so the second column is where a bump shows up in the diff
|
|
# 3. the SHA-256 of the archive in that release, in lowercase hex
|
|
#
|
|
# A build that fails on a mismatch is telling you that the bytes behind a
|
|
# pinned tag are not the bytes that were reviewed. A build that merely
|
|
# warns is telling you pgbuild has published something newer, which is
|
|
# routine and is not an error: the build carries on with the pinned
|
|
# version. Either way the fix is to refresh, read the diff and commit it:
|
|
#
|
|
# tools/refresh_pgbuild_lock.py <package>
|
|
#
|
|
# This file is generated by that script, so edit it through the script
|
|
# rather than by hand.
|
|
"""
|
|
|
|
|
|
def rolling_key(package, platform, arch, major=None):
|
|
"""Return the name the action looks an entry up by.
|
|
|
|
This is the rolling tag without its -latest suffix, which is what the
|
|
action can work out for itself from the package, the platform and the
|
|
architecture it was asked for.
|
|
"""
|
|
name = package if major is None else '%s-%s' % (package, major)
|
|
|
|
return '%s-%s-%s' % (name, platform, arch)
|
|
|
|
|
|
def archive_name(tag, platform):
|
|
"""Return the asset name pgbuild publishes in a release."""
|
|
return tag + ('.tar.gz' if platform == 'macos' else '.zip')
|
|
|
|
|
|
def expected_entries():
|
|
"""Every dependency the action can fetch, grouped for the lock file.
|
|
|
|
Returns a list of (heading, [entry, ...]) in the order the sections are
|
|
written out, each entry a dict describing one dependency.
|
|
"""
|
|
sections = []
|
|
|
|
for platform, packages, arches in (
|
|
('macos', MACOS_PACKAGES, MACOS_ARCHES),
|
|
('windows', WINDOWS_PACKAGES, WINDOWS_ARCHES),
|
|
):
|
|
for arch in arches:
|
|
entries = []
|
|
|
|
for package in sorted(packages):
|
|
majors = POSTGRES_MAJORS if package == 'postgresql' \
|
|
else [None]
|
|
|
|
for major in majors:
|
|
entries.append({
|
|
'package': package,
|
|
'platform': platform,
|
|
'arch': arch,
|
|
'major': major,
|
|
'key': rolling_key(package, platform, arch, major),
|
|
})
|
|
|
|
heading = '%s, %s' % (
|
|
'macOS' if platform == 'macos' else 'Windows', arch)
|
|
sections.append((heading, entries))
|
|
|
|
return sections
|
|
|
|
|
|
def read_lock(path):
|
|
"""Read the dependency to (tag, digest) mapping out of a lock file."""
|
|
pins = {}
|
|
|
|
if not os.path.exists(path):
|
|
return pins
|
|
|
|
with open(path, encoding='utf-8') as lock:
|
|
for line in lock:
|
|
line = line.split('#', 1)[0].strip()
|
|
|
|
if not line:
|
|
continue
|
|
|
|
fields = line.split()
|
|
|
|
if len(fields) != 3:
|
|
raise SystemExit(
|
|
'%s: cannot parse %r; expected a dependency, a tag and '
|
|
'a digest' % (path, line))
|
|
|
|
pins[fields[0]] = (fields[1], fields[2])
|
|
|
|
return pins
|
|
|
|
|
|
def manifest_versions():
|
|
"""Return the upstream version pgbuild builds of each package.
|
|
|
|
manifest.json is what pgbuild builds from and what its release tags are
|
|
named after, so it answers the question of which version to pin without
|
|
anybody having to read a tag list and guess.
|
|
"""
|
|
try:
|
|
with urllib.request.urlopen(MANIFEST_URL) as response:
|
|
manifest = json.load(response)
|
|
except (urllib.error.URLError, ValueError) as exc:
|
|
raise SystemExit('Failed to read %s: %s' % (MANIFEST_URL, exc))
|
|
|
|
return {package['name']: package['version']
|
|
for package in manifest['packages']}
|
|
|
|
|
|
def postgres_version(versions, major):
|
|
"""The full PostgreSQL version pgbuild builds for one major.
|
|
|
|
The manifest lists every supported major's current version, and pgbuild
|
|
tags the release with the full one, so 18 has to be resolved to 18.6
|
|
rather than guessed at.
|
|
"""
|
|
# The manifest records PostgreSQL's supported versions as one string
|
|
# holding a list, which is what the workflows hand to fromJSON, so the
|
|
# versions are picked out of it rather than indexed.
|
|
listed = versions.get('postgresql') or ''
|
|
|
|
if isinstance(listed, str):
|
|
listed = re.findall(r'\d+(?:\.\d+)+', listed)
|
|
|
|
for version in listed:
|
|
if str(version).split('.')[0] == str(major):
|
|
return version
|
|
|
|
raise SystemExit(
|
|
'pgbuild\'s manifest lists no PostgreSQL %s, so there is no version '
|
|
'to pin to' % major)
|
|
|
|
|
|
def tag_for(entry, versions):
|
|
"""Return the release tag to pin one dependency to.
|
|
|
|
Everything, PostgreSQL included, is tagged with the full upstream
|
|
version pgbuild's manifest names, so a PostgreSQL major has to be
|
|
resolved to the version built for it: the rolling tag says 18, whilst
|
|
the release that never changes says 18.6.
|
|
"""
|
|
if entry['package'] == 'postgresql':
|
|
version = postgres_version(versions, entry['major'])
|
|
else:
|
|
version = versions.get(entry['package'])
|
|
|
|
if version is None:
|
|
raise SystemExit(
|
|
'%s is not in pgbuild\'s manifest, so there is no version '
|
|
'to pin it to' % entry['package'])
|
|
|
|
return '%s-%s-%s-%s' % (entry['package'], version, entry['platform'],
|
|
entry['arch'])
|
|
|
|
|
|
def download_digest(tag, platform):
|
|
"""Hash one archive, returning None where the release does not exist.
|
|
|
|
The archives run to tens of megabytes and nothing here needs their
|
|
contents, so they are hashed as they stream past rather than written
|
|
to a temporary file somebody would then have to clean up.
|
|
"""
|
|
archive = archive_name(tag, platform)
|
|
url = '%s/%s/%s' % (BASE_URL, tag, archive)
|
|
digest = hashlib.sha256()
|
|
|
|
try:
|
|
with urllib.request.urlopen(url) as response:
|
|
while True:
|
|
chunk = response.read(1024 * 1024)
|
|
|
|
if not chunk:
|
|
break
|
|
|
|
digest.update(chunk)
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
|
|
raise SystemExit('Failed to fetch %s: %s' % (url, exc))
|
|
except urllib.error.URLError as exc:
|
|
raise SystemExit('Failed to fetch %s: %s' % (url, exc))
|
|
|
|
return digest.hexdigest()
|
|
|
|
|
|
def digest_for(entry, tag):
|
|
"""Hash the archive a dependency is pinned to.
|
|
|
|
A tag that does not exist is an error rather than something to work
|
|
around: the whole point of pinning is that the lock names an archive
|
|
which will still be there months from now, so a missing one means
|
|
pgbuild has not published that version and the lock would be recording
|
|
something nobody can fetch.
|
|
"""
|
|
digest = download_digest(tag, entry['platform'])
|
|
|
|
if digest is None:
|
|
raise SystemExit(
|
|
'%s does not exist in pgbuild, so there is nothing to pin. If '
|
|
'its build has not finished yet, wait for it and try again.'
|
|
% tag)
|
|
|
|
return digest
|
|
|
|
|
|
def write_lock(path, sections, pins):
|
|
"""Write the lock file out, aligned and in a stable order."""
|
|
keys = [entry['key'] for _, entries in sections for entry in entries]
|
|
tags = [pins[key][0] for key in keys]
|
|
key_width = max(len(key) for key in keys) + 2
|
|
tag_width = max(len(tag) for tag in tags) + 2
|
|
|
|
with open(path, 'w', encoding='utf-8') as lock:
|
|
lock.write(HEADER)
|
|
|
|
for heading, entries in sections:
|
|
lock.write('\n# %s\n' % heading)
|
|
|
|
for entry in entries:
|
|
tag, digest = pins[entry['key']]
|
|
lock.write('%s%s%s\n' % (
|
|
entry['key'].ljust(key_width), tag.ljust(tag_width),
|
|
digest))
|
|
|
|
|
|
def parse_pins(values):
|
|
"""Parse the --tag DEPENDENCY=TAG overrides into a mapping."""
|
|
overrides = {}
|
|
|
|
for value in values:
|
|
if '=' not in value:
|
|
raise SystemExit(
|
|
'Expected --tag DEPENDENCY=TAG, not %r' % value)
|
|
|
|
key, tag = value.split('=', 1)
|
|
overrides[key.strip()] = tag.strip()
|
|
|
|
return overrides
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description='Refresh the pgbuild dependency lock file.')
|
|
parser.add_argument(
|
|
'packages', nargs='*', metavar='PACKAGE',
|
|
help='packages to refresh, e.g. openssl; all of them if omitted')
|
|
parser.add_argument(
|
|
'--tag', action='append', default=[], metavar='DEPENDENCY=TAG',
|
|
help='pin one dependency to a named tag rather than to whatever '
|
|
'version pgbuild\'s manifest currently names')
|
|
parser.add_argument(
|
|
'--lock', default=None,
|
|
help='path to the lock file (default: pkg/pgbuild-deps.lock)')
|
|
args = parser.parse_args()
|
|
|
|
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
path = args.lock or os.path.join(root, 'pkg', 'pgbuild-deps.lock')
|
|
|
|
sections = expected_entries()
|
|
known = sorted({entry['package'] for _, entries in sections
|
|
for entry in entries})
|
|
|
|
unknown = [name for name in args.packages if name not in known]
|
|
|
|
if unknown:
|
|
raise SystemExit('Unknown package(s): %s. Known packages are: %s'
|
|
% (', '.join(unknown), ', '.join(known)))
|
|
|
|
overrides = parse_pins(args.tag)
|
|
pins = read_lock(path)
|
|
todo = []
|
|
|
|
for _, entries in sections:
|
|
for entry in entries:
|
|
# A dependency with no pin yet has to be resolved whatever was
|
|
# asked for, since leaving it out would write a lock file the
|
|
# action then refuses to build against.
|
|
wanted = not args.packages or entry['package'] in args.packages
|
|
|
|
if wanted or entry['key'] not in pins:
|
|
todo.append(entry)
|
|
|
|
if not todo:
|
|
raise SystemExit('Nothing to do.')
|
|
|
|
versions = manifest_versions()
|
|
refreshed = 0
|
|
|
|
for entry in todo:
|
|
key = entry['key']
|
|
tag = overrides.get(key) or tag_for(entry, versions)
|
|
|
|
print('Hashing %s...' % archive_name(tag, entry['platform']))
|
|
digest = digest_for(entry, tag)
|
|
|
|
if pins.get(key) != (tag, digest):
|
|
refreshed += 1
|
|
|
|
pins[key] = (tag, digest)
|
|
|
|
write_lock(path, sections, pins)
|
|
|
|
print('\nWrote %s, %d entr%s changed.'
|
|
% (path, refreshed, 'y' if refreshed == 1 else 'ies'))
|
|
print('Review the diff before committing it: it is the record of a '
|
|
'change to what we sign.')
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|