mirror of
https://github.com/quay/quay.git
synced 2026-01-26 06:21:37 +03:00
f3c3916bc7
Having only one worker index recent manifest doesn't keep up with the rate new manifests being pushed, given the time it takes for an index request to complete. Adding the option to bypass the global lock allows for more workers, but also increase the chance of duplicate work.
94 lines
3.0 KiB
Python
94 lines
3.0 KiB
Python
import logging.config
|
|
import os
|
|
import time
|
|
|
|
import features
|
|
import threading
|
|
|
|
from app import app
|
|
from data.secscan_model import secscan_model
|
|
from workers.gunicorn_worker import GunicornWorker
|
|
from workers.worker import Worker
|
|
from util.locking import GlobalLock, LockNotAcquiredException
|
|
from util.log import logfile_path
|
|
from endpoints.v2 import v2_bp
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
DEFAULT_INDEXING_INTERVAL = 30
|
|
|
|
|
|
class SecurityWorker(Worker):
|
|
def __init__(self):
|
|
super(SecurityWorker, self).__init__()
|
|
self._next_token = None
|
|
self._model = secscan_model
|
|
|
|
interval = app.config.get("SECURITY_SCANNER_INDEXING_INTERVAL", DEFAULT_INDEXING_INTERVAL)
|
|
self.add_operation(self._index_in_scanner, interval)
|
|
self.add_operation(self._index_recent_manifests_in_scanner, interval)
|
|
|
|
def _index_in_scanner(self):
|
|
batch_size = app.config.get("SECURITY_SCANNER_V4_BATCH_SIZE", 0)
|
|
self._next_token = self._model.perform_indexing(self._next_token, batch_size)
|
|
|
|
def _index_recent_manifests_in_scanner(self):
|
|
batch_size = app.config.get("SECURITY_SCANNER_V4_RECENT_MANIFEST_BATCH_SIZE", 1000)
|
|
|
|
if not app.config.get("SECURITY_SCANNER_V4_SKIP_RECENT_MANIFEST_BATCH_LOCK", False):
|
|
try:
|
|
with GlobalLock(
|
|
"SECURITYWORKER_INDEX_RECENT_MANIFEST", lock_ttl=300, auto_renewal=True
|
|
):
|
|
self._model.perform_indexing_recent_manifests(batch_size)
|
|
except LockNotAcquiredException:
|
|
logger.warning(
|
|
"Could not acquire global lock for recent manifest indexing. Skipping"
|
|
)
|
|
|
|
else:
|
|
self._model.perform_indexing_recent_manifests(batch_size)
|
|
|
|
|
|
def create_gunicorn_worker():
|
|
"""
|
|
follows the gunicorn application factory pattern, enabling
|
|
a quay worker to run as a gunicorn worker thread.
|
|
|
|
this is useful when utilizing gunicorn's hot reload in local dev.
|
|
|
|
utilizing this method will enforce a 1:1 quay worker to gunicorn worker ratio.
|
|
"""
|
|
app.register_blueprint(v2_bp, url_prefix="/v2")
|
|
worker = GunicornWorker(__name__, app, SecurityWorker(), features.SECURITY_SCANNER)
|
|
return worker
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if os.getenv("PYDEV_DEBUG", None):
|
|
import pydevd_pycharm
|
|
|
|
host, port = os.getenv("PYDEV_DEBUG").split(":")
|
|
pydevd_pycharm.settrace(
|
|
host, port=int(port), stdoutToServer=True, stderrToServer=True, suspend=False
|
|
)
|
|
|
|
app.register_blueprint(v2_bp, url_prefix="/v2")
|
|
|
|
if app.config.get("ACCOUNT_RECOVERY_MODE", False):
|
|
logger.debug("Quay running in account recovery mode")
|
|
while True:
|
|
time.sleep(100000)
|
|
|
|
if not features.SECURITY_SCANNER:
|
|
logger.debug("Security scanner disabled; skipping SecurityWorker")
|
|
while True:
|
|
time.sleep(100000)
|
|
|
|
GlobalLock.configure(app.config)
|
|
logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False)
|
|
worker = SecurityWorker()
|
|
worker.start()
|