mirror of
https://github.com/quay/quay.git
synced 2025-04-18 10:44:06 +03:00
a839a78eb5
Adds ACCOUNT_RECOVERY_MODE to allow Quay to run with some core features disabled. When this is set, the instance should only be used in order by existing users who hasn't linked their account to an external login service, after database authentication has been disabled.
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
import logging
|
|
import logging.config
|
|
import time
|
|
|
|
from datetime import timedelta, datetime
|
|
|
|
from app import app, storage
|
|
from data.database import UseThenDisconnect
|
|
from workers.blobuploadcleanupworker.models_pre_oci import pre_oci_model as model
|
|
from workers.worker import Worker
|
|
from util.log import logfile_path
|
|
from util.locking import GlobalLock, LockNotAcquiredException
|
|
from workers.gunicorn_worker import GunicornWorker
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DELETION_DATE_THRESHOLD = timedelta(days=2)
|
|
BLOBUPLOAD_CLEANUP_FREQUENCY = app.config.get("BLOBUPLOAD_CLEANUP_FREQUENCY", 60 * 60)
|
|
LOCK_TTL = 60 * 20 # 20 minutes
|
|
|
|
|
|
class BlobUploadCleanupWorker(Worker):
|
|
def __init__(self):
|
|
super(BlobUploadCleanupWorker, self).__init__()
|
|
self.add_operation(self._try_cleanup_uploads, BLOBUPLOAD_CLEANUP_FREQUENCY)
|
|
|
|
def _try_cleanup_uploads(self):
|
|
"""
|
|
Performs garbage collection on the blobupload table.
|
|
"""
|
|
try:
|
|
with GlobalLock("BLOB_CLEANUP", lock_ttl=LOCK_TTL):
|
|
self._cleanup_uploads()
|
|
except LockNotAcquiredException:
|
|
logger.debug("Could not acquire global lock for blob upload cleanup worker")
|
|
return
|
|
|
|
def _cleanup_uploads(self):
|
|
"""
|
|
Performs cleanup on the blobupload table.
|
|
"""
|
|
logger.debug("Performing blob upload cleanup")
|
|
|
|
while True:
|
|
# Find all blob uploads older than the threshold (typically a week) and delete them.
|
|
with UseThenDisconnect(app.config):
|
|
stale_upload = model.get_stale_blob_upload(DELETION_DATE_THRESHOLD)
|
|
if stale_upload is None:
|
|
logger.debug("No additional stale blob uploads found")
|
|
return
|
|
|
|
# Remove the stale upload from storage.
|
|
logger.debug("Removing stale blob upload %s", stale_upload.uuid)
|
|
assert stale_upload.created <= (datetime.utcnow() - DELETION_DATE_THRESHOLD)
|
|
|
|
try:
|
|
storage.cancel_chunked_upload(
|
|
[stale_upload.location_name], stale_upload.uuid, stale_upload.storage_metadata
|
|
)
|
|
except Exception as ex:
|
|
logger.debug(
|
|
"Got error when trying to cancel chunked upload %s: %s",
|
|
stale_upload.uuid,
|
|
str(ex),
|
|
)
|
|
|
|
# Delete the stale upload's row.
|
|
with UseThenDisconnect(app.config):
|
|
model.delete_blob_upload(stale_upload)
|
|
|
|
logger.debug("Removed stale blob upload %s", stale_upload.uuid)
|
|
|
|
|
|
def create_gunicorn_worker():
|
|
"""
|
|
follows the gunicorn application factory pattern, enabling
|
|
a quay worker to run as a gunicorn worker thread.
|
|
|
|
this is useful when utilizing gunicorn's hot reload in local dev.
|
|
|
|
utilizing this method will enforce a 1:1 quay worker to gunicorn worker ratio.
|
|
"""
|
|
worker = GunicornWorker(__name__, app, BlobUploadCleanupWorker(), True)
|
|
return worker
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if app.config.get("ACCOUNT_RECOVERY_MODE", False):
|
|
logger.debug("Quay running in account recovery mode")
|
|
while True:
|
|
time.sleep(100000)
|
|
|
|
logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False)
|
|
GlobalLock.configure(app.config)
|
|
worker = BlobUploadCleanupWorker()
|
|
worker.start()
|