mirror of
https://github.com/quay/quay.git
synced 2026-01-26 06:21:37 +03:00
38be6d05d0
* Convert all Python2 to Python3 syntax. * Removes oauth2lib dependency * Replace mockredis with fakeredis * byte/str conversions * Removes nonexisting __nonzero__ in Python3 * Python3 Dockerfile and related * [PROJQUAY-98] Replace resumablehashlib with rehash * PROJQUAY-123 - replace gpgme with python3-gpg * [PROJQUAY-135] Fix unhashable class error * Update external dependencies for Python 3 - Move github.com/app-registry/appr to github.com/quay/appr - github.com/coderanger/supervisor-stdout - github.com/DevTable/container-cloud-config - Update to latest mockldap with changes applied from coreos/mockldap - Update dependencies in requirements.txt and requirements-dev.txt * Default FLOAT_REPR function to str in json encoder and removes keyword assignment True, False, and str were not keywords in Python2... * [PROJQUAY-165] Replace package `bencode` with `bencode.py` - Bencode is not compatible with Python 3.x and is no longer maintained. Bencode.py appears to be a drop-in replacement/fork that is compatible with Python 3. * Make sure monkey.patch is called before anything else ( * Removes anunidecode dependency and replaces it with text_unidecode * Base64 encode/decode pickle dumps/loads when storing value in DB Base64 encodes/decodes the serialized values when storing them in the DB. Also make sure to return a Python3 string instead of a Bytes when coercing for db, otherwise, Postgres' TEXT field will convert it into a hex representation when storing the value. * Implement __hash__ on Digest class In Python 3, if a class defines __eq__() but not __hash__(), its instances will not be usable as items in hashable collections (e.g sets). * Remove basestring check * Fix expected message in credentials tests * Fix usage of Cryptography.Fernet for Python3 (#219) - Specifically, this addresses the issue where Byte<->String conversions weren't being applied correctly. * Fix utils - tar+stream layer format utils - filelike util * Fix storage tests * Fix endpoint tests * Fix workers tests * Fix docker's empty layer bytes * Fix registry tests * Appr * Enable CI for Python 3.6 * Skip buildman tests Skip buildman tests while it's being rewritten to allow ci to pass. * Install swig for CI * Update expected exception type in redis validation test * Fix gpg signing calls Fix gpg calls for updated gpg wrapper, and add signing tests. * Convert / to // for Python3 integer division * WIP: Update buildman to use asyncio instead of trollius. This dependency is considered deprecated/abandoned and was only used as an implementation/backport of asyncio on Python 2.x This is a work in progress, and is included in the PR just to get the rest of the tests passing. The builder is actually being rewritten. * Target Python 3.8 * Removes unused files - Removes unused files that were added accidentally while rebasing - Small fixes/cleanup - TODO tasks comments * Add TODO to verify rehash backward compat with resumablehashlib * Revert "[PROJQUAY-135] Fix unhashable class error" and implements __hash__ instead. This reverts commit 735e38e3c1d072bf50ea864bc7e119a55d3a8976. Instead, defines __hash__ for encryped fields class, using the parent field's implementation. * Remove some unused files ad imports Co-authored-by: Kenny Lee Sin Cheong <kenny.lee@redhat.com> Co-authored-by: Tom McKay <thomasmckay@redhat.com>
103 lines
2.9 KiB
Python
103 lines
2.9 KiB
Python
from data.database import (
|
|
ImageStorage,
|
|
Image,
|
|
ImageStoragePlacement,
|
|
ImageStorageLocation,
|
|
RepositoryTag,
|
|
)
|
|
from data import model
|
|
from app import storage as storage_system
|
|
from tqdm import tqdm
|
|
|
|
|
|
def find_broken_storages():
|
|
broken_storages = set()
|
|
|
|
print("Checking storages...")
|
|
placement_count = ImageStoragePlacement.select().count()
|
|
placements = (
|
|
ImageStoragePlacement.select()
|
|
.join(ImageStorage)
|
|
.switch(ImageStoragePlacement)
|
|
.join(ImageStorageLocation)
|
|
)
|
|
|
|
for placement in tqdm(placements, total=placement_count):
|
|
path = model.storage.get_layer_path(placement.storage)
|
|
if not storage_system.exists([placement.location.name], path):
|
|
broken_storages.add(placement.storage.id)
|
|
|
|
return list(broken_storages)
|
|
|
|
|
|
def delete_broken_layers():
|
|
result = input('Please make sure your registry is not running and enter "GO" to continue: ')
|
|
if result != "GO":
|
|
print("Declined to run")
|
|
return
|
|
|
|
broken_storages = find_broken_storages()
|
|
if not broken_storages:
|
|
print("No broken layers found")
|
|
return
|
|
|
|
# Find all the images referencing the broken layers.
|
|
print("Finding broken images...")
|
|
IMAGE_BATCH_SIZE = 100
|
|
|
|
all_images = []
|
|
for i in tqdm(list(range(0, len(broken_storages) / IMAGE_BATCH_SIZE))):
|
|
start = i * IMAGE_BATCH_SIZE
|
|
end = (i + 1) * IMAGE_BATCH_SIZE
|
|
|
|
images = (
|
|
Image.select().join(ImageStorage).where(Image.storage << broken_storages[start:end])
|
|
)
|
|
all_images.extend(images)
|
|
|
|
if not all_images:
|
|
print("No broken layers found")
|
|
return
|
|
|
|
# Find all the tags containing the images.
|
|
print("Finding associated tags for %s images..." % len(all_images))
|
|
all_tags = {}
|
|
for image in tqdm(all_images):
|
|
query = model.tag.get_matching_tags(
|
|
image.docker_image_id, image.storage.uuid, RepositoryTag
|
|
)
|
|
for tag in query:
|
|
all_tags[tag.id] = tag
|
|
|
|
# Ask to delete them.
|
|
print("")
|
|
print("The following tags were found to reference invalid images:")
|
|
for tag in list(all_tags.values()):
|
|
print("%s/%s: %s" % (tag.repository.namespace_user.username, tag.repository.name, tag.name))
|
|
|
|
if not all_tags:
|
|
print("(Tags in time machine)")
|
|
|
|
print("")
|
|
result = input(
|
|
'Enter "DELETENOW" to delete these tags and ALL associated images (THIS IS PERMANENT): '
|
|
)
|
|
if result != "DELETENOW":
|
|
print("Declined to delete")
|
|
return
|
|
|
|
print("")
|
|
print("Marking tags to be GCed...")
|
|
for tag in tqdm(list(all_tags.values())):
|
|
tag.lifetime_end_ts = 0
|
|
tag.save()
|
|
|
|
print("GCing all repositories...")
|
|
for tag in tqdm(list(all_tags.values())):
|
|
model.repository.garbage_collect_repo(tag.repository)
|
|
|
|
print("All done! You may now restart your registry.")
|
|
|
|
|
|
delete_broken_layers()
|