mirror of
https://github.com/quay/quay.git
synced 2026-01-26 06:21:37 +03:00
fca67e7729
* Add dev dependencies mypy and typing * Add makefile target `types-test`, not yet included in `test` target. * Generate stubs for imported modules to avoid mypy complaining about missing types. * Remove generated stubs as there are way too many and they cause tons of mess in the repo. Switched to ignoring untyped modules for now, to concentrate on Quay-only type checking. * mypy config changed to ignore missing imports * ignore property decorator as it is not supported by mypy * mypy annotations for many configuration variables * re-generate mypy_stubs directory as its necessary in some classes for base classes to prevent mypy errors * util/registry/queuefile referred to non existent definition of Empty class in multiprocessing.queues * ignore type checking for things like monkey patching and exported/re-imported objects that mypy does not allow. * Adjust mypy config to warn us about unreachable return paths and useless expressions. * Add the __annotations__ property to INTERNAL_ONLY_PROPERTIES so that it is not part of the config schema testing * Remove redundant dependencies `typing` and `typing-extensions` which are NOOP after Python 3.5 * Remove mypy-extensions which only provides a TypedDict implementation but has not been updated since 2019. * updated mypy to 0.910 which requires all types packages to be installed manually. * exclude local-dev from type checking until core team can suggest an outcome for __init__.py duplicate packages * re-add typing dependency which will be needed until Python 3.9 * ignore .mypy_cache * add mypy stub for features module to replace inline definitions * import annotations eager evaluation in billing.py as it was required to reference a class declared later in the module. * remove the type definition of V1ProtocolSteps/V2ProtocolSteps to make tox happy
85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
"""
|
|
Usage (from the root in the container): venv/bin/python -m util.verifyplacements.
|
|
|
|
This script verifies that if a blob is listed as being in a specific storage location, the file
|
|
actually exists there. If the file is not found in that storage location, the placement entry in the
|
|
database is removed.
|
|
"""
|
|
|
|
from typing import Dict
|
|
import logging
|
|
|
|
from peewee import fn
|
|
|
|
from app import storage
|
|
from data import model
|
|
from data.database import ImageStorage, ImageStoragePlacement, ImageStorageLocation
|
|
from util.migrate.allocator import yield_random_entries
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
LOCATION_MAP: Dict[str, ImageStorageLocation] = {}
|
|
|
|
|
|
def _get_location_row(location):
|
|
if location in LOCATION_MAP:
|
|
return LOCATION_MAP[location]
|
|
|
|
location_row = ImageStorageLocation.get(name=location)
|
|
LOCATION_MAP[location] = location_row
|
|
return location_row
|
|
|
|
|
|
def verify_placements():
|
|
encountered = set()
|
|
|
|
iterator = yield_random_entries(
|
|
lambda: ImageStorage.select(),
|
|
ImageStorage.id,
|
|
1000,
|
|
ImageStorage.select(fn.Max(ImageStorage.id)).scalar(),
|
|
1,
|
|
)
|
|
|
|
for storage_row, abt, _ in iterator:
|
|
if storage_row.id in encountered:
|
|
continue
|
|
|
|
encountered.add(storage_row.id)
|
|
|
|
logger.info("Checking placements for storage `%s`", storage_row.uuid)
|
|
try:
|
|
with_locations = model.storage.get_storage_by_uuid(storage_row.uuid)
|
|
except model.InvalidImageException:
|
|
logger.exception("Could not find storage `%s`", storage_row.uuid)
|
|
continue
|
|
|
|
storage_path = model.storage.get_layer_path(storage_row)
|
|
locations_to_check = set(with_locations.locations)
|
|
if locations_to_check:
|
|
logger.info(
|
|
"Checking locations `%s` for storage `%s`", locations_to_check, storage_row.uuid
|
|
)
|
|
for location in locations_to_check:
|
|
logger.info("Checking location `%s` for storage `%s`", location, storage_row.uuid)
|
|
if not storage.exists([location], storage_path):
|
|
location_row = _get_location_row(location)
|
|
logger.info(
|
|
"Location `%s` is missing for storage `%s`; removing",
|
|
location,
|
|
storage_row.uuid,
|
|
)
|
|
(
|
|
ImageStoragePlacement.delete()
|
|
.where(
|
|
ImageStoragePlacement.storage == storage_row,
|
|
ImageStoragePlacement.location == location_row,
|
|
)
|
|
.execute()
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO)
|
|
verify_placements()
|