mirror of
https://github.com/quay/quay.git
synced 2026-01-26 06:21:37 +03:00
5f63b3a7bb
* chore: drop deprecated tables and remove unused code * isort imports * migration: check for table existence before drop
85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
"""
|
|
Usage (from the root in the container): venv/bin/python -m util.verifyplacements.
|
|
|
|
This script verifies that if a blob is listed as being in a specific storage location, the file
|
|
actually exists there. If the file is not found in that storage location, the placement entry in the
|
|
database is removed.
|
|
"""
|
|
|
|
import logging
|
|
from typing import Dict
|
|
|
|
from peewee import fn
|
|
|
|
from app import storage
|
|
from data import model
|
|
from data.database import ImageStorage, ImageStorageLocation, ImageStoragePlacement
|
|
from util.migrate.allocator import yield_random_entries
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
LOCATION_MAP: Dict[str, ImageStorageLocation] = {}
|
|
|
|
|
|
def _get_location_row(location):
|
|
if location in LOCATION_MAP:
|
|
return LOCATION_MAP[location]
|
|
|
|
location_row = ImageStorageLocation.get(name=location)
|
|
LOCATION_MAP[location] = location_row
|
|
return location_row
|
|
|
|
|
|
def verify_placements():
|
|
encountered = set()
|
|
|
|
iterator = yield_random_entries(
|
|
lambda: ImageStorage.select(),
|
|
ImageStorage.id,
|
|
1000,
|
|
ImageStorage.select(fn.Max(ImageStorage.id)).scalar(),
|
|
1,
|
|
)
|
|
|
|
for storage_row, abt, _ in iterator:
|
|
if storage_row.id in encountered:
|
|
continue
|
|
|
|
encountered.add(storage_row.id)
|
|
|
|
logger.info("Checking placements for storage `%s`", storage_row.uuid)
|
|
try:
|
|
with_locations = model.storage.get_storage_by_uuid(storage_row.uuid)
|
|
except model.InvalidImageException:
|
|
logger.exception("Could not find storage `%s`", storage_row.uuid)
|
|
continue
|
|
|
|
storage_path = model.storage.get_layer_path(storage_row)
|
|
locations_to_check = set(with_locations.locations)
|
|
if locations_to_check:
|
|
logger.info(
|
|
"Checking locations `%s` for storage `%s`", locations_to_check, storage_row.uuid
|
|
)
|
|
for location in locations_to_check:
|
|
logger.info("Checking location `%s` for storage `%s`", location, storage_row.uuid)
|
|
if not storage.exists([location], storage_path):
|
|
location_row = _get_location_row(location)
|
|
logger.info(
|
|
"Location `%s` is missing for storage `%s`; removing",
|
|
location,
|
|
storage_row.uuid,
|
|
)
|
|
(
|
|
ImageStoragePlacement.delete()
|
|
.where(
|
|
ImageStoragePlacement.storage == storage_row,
|
|
ImageStoragePlacement.location == location_row,
|
|
)
|
|
.execute()
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO)
|
|
verify_placements()
|