docker/quay
docker/quay
1
0
Fork 0
mirror of https://github.com/quay/quay.git synced 2025-04-18 10:44:06 +03:00
Code
quay/Makefile
Ivan Bazulic 9fa48ea376
nginx: Increase the number and size of proxy buffers (PROJQUAY-6950) (#3303) 
* nginx: Increase the number and size of proxy buffers (PROJQUAY-6950)
From [nginx documentation](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering):

> When buffering is enabled, nginx receives a response from the proxied server as soon as possible, saving it into the buffers set by the proxy_buffer_size and proxy_buffers directives.
> If the whole response does not fit into memory, a part of it can be saved to a temporary file on the disk.
> Writing to temporary files is controlled by the proxy_max_temp_file_size and proxy_temp_file_write_size directives.
>
> When buffering is disabled, the response is passed to a client synchronously, immediately as it is received. nginx will not try to read the whole response from the proxied server.
> The maximum size of the data that nginx can receive from the server at a time is set by the proxy_buffer_size directive.

By default, the value of `proxy_buffer_size` directive, if not set in the nginx configuration, is equal to one memory page which on most platforms equals 4 KiB of memory. When `FEATURE_PROXY_STORAGE` is turned on and STS driver is used, the size of headers reaches that limit of 4 KiB and, if it surpasses it, nginx will error out and will not process the request. With this PR we set the buffers to an adequate size so that proxy requests are properly processed by nginx.

* Add additional nginx directive to make buffers work
2024-10-04 11:31:48 -04:00

270 lines
9.3 KiB
Makefile
Raw Permalink Blame History

SHELL := /bin/bash
DOCKER ?= docker
DOCKER_COMPOSE ?= $(DOCKER) compose
export PATH := ./venv/bin:$(PATH)
SHA := $(shell git rev-parse --short HEAD )
REPO := quay.io/quay/quay
TAG := $(REPO):$(SHA)
MODIFIED_FILES_COUNT = $(shell git diff --name-only origin/master | grep -E .+\.py$ | wc -l)
GIT_MERGE_BASED = $(shell git merge-base origin/master HEAD)
MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.py$ | paste -sd ' ')
show-modified:
echo $(MODIFIED_FILES)
.PHONY: all unit-test registry-test registry-test-old buildman-test test build run clean
all: clean test build
unit-test:
TEST=true PYTHONPATH="." py.test \
--cov="." --cov-report=html --cov-report=term-missing \
-m 'not e2e' --timeout=3600 --verbose -x --ignore=buildman/ \
./
e2e-test:
TEST=true PYTHONPATH="." py.test \
--cov="." --cov-report=html --cov-report=term-missing \
-m 'e2e' --timeout=3600 --verbose -x --ignore=buildman/ \
./
integration-test:
TEST=true PYTHONPATH="." py.test \
--verbose --ignore=buildman/ \
test/integration/*
registry-test:
TEST=true PYTHONPATH="." py.test \
--cov="." --cov-report=html --cov-report=term-missing \
-m 'not e2e' --timeout=3600 --verbose -x \
test/registry/registry_tests.py
buildman-test:
TEST=true PYTHONPATH="." py.test \
--cov="." --cov-report=html --cov-report=term-missing \
-m 'not e2e' --timeout=3600 --verbose -x \
./buildman/
certs-test:
./test/test_certs_install.sh
full-db-test: ensure-test-db
TEST=true PYTHONPATH=. QUAY_OVERRIDE_CONFIG='{"DATABASE_SECRET_KEY": "anothercrazykey!"}' \
alembic upgrade head
TEST=true PYTHONPATH=. \
SKIP_DB_SCHEMA=true py.test -m 'not e2e' --timeout=7200 \
--verbose -x --ignore=endpoints/appr/test/ \
./
clients-test:
cd test/clients; python clients_test.py
types-test:
mypy .
test: unit-test registry-test registry-test-old certs-test
ensure-test-db:
@if [ -z $(TEST_DATABASE_URI) ]; then \
echo "TEST_DATABASE_URI is undefined"; \
exit 1; \
fi
install-pre-commit-hook:
pip install pre-commit==2.20.0
pre-commit install
PG_PASSWORD := quay
PG_USER := quay
PG_PORT := 5433
PG_HOST := postgresql://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/quay
CONTAINER := postgres-testrunner
TESTS ?= ./
test_postgres : TEST_ENV := SKIP_DB_SCHEMA=true TEST=true \
TEST_DATABASE_URI=$(PG_HOST) PYTHONPATH=.
test_postgres:
$(DOCKER) rm -f $(CONTAINER) || true
$(DOCKER) run --name $(CONTAINER) \
-e POSTGRES_PASSWORD=$(PG_PASSWORD) -e POSTGRES_USER=$(PG_USER) \
-p $(PG_PORT):5432 -d postgres:12.1
$(DOCKER) exec -it $(CONTAINER) bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
$(DOCKER) exec -it $(CONTAINER) bash -c "psql -U $(PG_USER) -d quay -c 'CREATE EXTENSION pg_trgm;'"
$(TEST_ENV) alembic upgrade head
$(TEST_ENV) py.test --timeout=7200 --verbose --ignore=endpoints/appr/test/ -x $(TESTS)
$(DOCKER) rm -f $(CONTAINER) || true
WEBPACK := node_modules/.bin/webpack
$(WEBPACK): package.json
npm install webpack
npm install
BUNDLE := static/js/build/bundle.js
$(BUNDLE): $(WEBPACK) tsconfig.json webpack.config.js typings.json
$(WEBPACK)
GRUNT := grunt/node_modules/.bin/grunt
$(GRUNT): grunt/package.json
cd grunt && npm install
JS := quay-frontend.js quay-frontend.min.js template-cache.js
CSS := quay-frontend.css
DIST := $(addprefix static/dist/, $(JS) $(CSS) cachebusters.json)
$(DIST): $(GRUNT)
cd grunt && ../$(GRUNT)
build: $(WEBPACK) $(GRUNT)
docker-build: build
ifneq (0,$(shell git status --porcelain | awk 'BEGIN {print $N}'))
echo 'dirty build not supported - run `FORCE=true make clean` to remove'
exit 1
endif
# get named head (ex: branch, tag, etc..)
NAME = $(shell git rev-parse --abbrev-ref HEAD)
# checkout commit so .git/HEAD points to full sha (used in Dockerfile)
git checkout $(SHA)
$(DOCKER) build -t $(TAG) .
git checkout $(NAME)
echo $(TAG)
app-sre-docker-build:
$(BUILD_CMD) -t ${IMG} -f Dockerfile .
clean:
find . -name "*.pyc" -exec rm -rf {} \;
rm -rf node_modules 2> /dev/null
rm -rf grunt/node_modules 2> /dev/null
rm -rf dest 2> /dev/null
rm -rf dist 2> /dev/null
rm -rf .cache 2> /dev/null
rm -rf static/js/build
rm -rf static/build
rm -rf static/dist
rm -rf build
rm -rf conf/stack
rm -rf screenshots
generate-proto-py:
python -m grpc_tools.protoc -Ibuildman/buildman_pb --python_out=buildman/buildman_pb --grpc_python_out=buildman/buildman_pb buildman.proto
black:
black --line-length=100 --target-version=py39 --exclude "/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)/" .
#################################
# Local Development Environment #
#################################
.PHONY: build-image-local-dev-frontend
build-images:: build-image-local-dev-frontend
build-image-local-dev-frontend:
# $(DOCKER)-compose run does not build images, so we need to build them if needed
test -n "$$($(DOCKER) images localhost/quay-build:latest -q)" || $(DOCKER_COMPOSE) build local-dev-frontend
.PHONY: build-image-quay
build-images:: build-image-quay
build-image-quay: .build-image-quay-stamp
.build-image-quay-stamp: Dockerfile requirements.txt
$(DOCKER_COMPOSE) build quay
touch $@
node_modules: node_modules/.npm-install-stamp
node_modules/.npm-install-stamp: package.json package-lock.json | build-image-local-dev-frontend
DOCKER_USER="$$(id -u):$$(id -g)" $(DOCKER_COMPOSE) run --rm --name quay-local-dev-frontend-install --entrypoint="" local-dev-frontend npm install --ignore-engines
# if npm install fails for some reason, it may have already created
# node_modules, so we cannot rely on the directory timestamps and should mark
# successfull runs of npm install with a stamp file.
touch $@
.PHONY: local-dev-clean
local-dev-clean:
rm -f ./conf/jwtproxy_conf.yaml ./conf/mitm.cert ./conf/mitm.key ./conf/quay.kid ./conf/quay.pem ./conf/supervisord.conf
rm -rf ./conf/__pycache__ ./static/build
.PHONY: local-dev-build-frontend
local-dev-build:: local-dev-build-frontend
local-dev-build-frontend: node_modules
DOCKER_USER="$$(id -u):$$(id -g)" $(DOCKER_COMPOSE) run --rm --name quay-local-dev-frontend-build --entrypoint="" local-dev-frontend npm run build
.PHONY: local-dev-build-images
local-dev-build:: local-dev-build-images
local-dev-build-images:
$(DOCKER_COMPOSE) build
.PHONY: local-dev-up
local-dev-up: local-dev-clean node_modules | build-image-quay
DOCKER_USER="$$(id -u):$$(id -g)" $(DOCKER_COMPOSE) up -d --force-recreate local-dev-frontend
$(DOCKER_COMPOSE) up -d redis quay-db
$(DOCKER) exec -it quay-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
DOCKER_USER="$$(id -u):0" $(DOCKER_COMPOSE) stop quay # we need to restart quay after local-dev-clean
DOCKER_USER="$$(id -u):0" $(DOCKER_COMPOSE) up -d quay
# Waiting until the frontend is built...
# Use '$(DOCKER_COMPOSE) logs -f local-dev-frontend' to see the progress
while ! test -e ./static/build/main-quay-frontend.bundle.js; do sleep 2; done
@echo "You can now access the frontend at http://localhost:8080"
.PHONY: update-testdata
update-testdata: local-dev-clean node_modules | build-image-quay
$(DOCKER_COMPOSE) rm -fsv quay-db quay
$(DOCKER) volume rm -f quay_quay-db-data
$(DOCKER_COMPOSE) up -d redis quay-db
$(DOCKER) exec -it quay-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
cd ./web/ && npm run quay:seed-db
DOCKER_USER="$$(id -u):0" $(DOCKER_COMPOSE) up -d quay
cd ./web/ && npm run quay:seed-storage
while ! curl -fso /dev/null http://localhost:8080; do echo "waiting for quay"; sleep 2; done
$(DOCKER) exec -it quay-db psql quay -U quay -c " \
DELETE FROM servicekey; \
DELETE FROM servicekeyapproval; \
SELECT pg_catalog.setval('public.notification_id_seq', 1, false); \
SELECT pg_catalog.setval('public.servicekey_id_seq', 1, false); \
SELECT pg_catalog.setval('public.servicekeyapproval_id_seq', 1, false); \
"
cd ./web/ && npm run quay:dump
$(DOCKER_COMPOSE) down
local-docker-rebuild:
$(DOCKER_COMPOSE) up -d --build redis
$(DOCKER_COMPOSE) up -d --build quay-db
$(DOCKER) exec -it quay-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
DOCKER_USER="$$(id -u):0" $(DOCKER_COMPOSE) up -d --build quay
$(DOCKER_COMPOSE) restart quay
ifeq ($(CLAIR),true)
$(DOCKER_COMPOSE) up -d --build clair-db
$(DOCKER) exec -it clair-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
$(DOCKER_COMPOSE) up -d --build clair
$(DOCKER_COMPOSE) restart clair
else
@echo "Skipping Clair"
endif
.PHONY: local-dev-up-with-clair
local-dev-up-with-clair: local-dev-up
$(DOCKER_COMPOSE) up -d clair-db
$(DOCKER) exec -it clair-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
DOCKER_USER="$$(id -u):0" $(DOCKER_COMPOSE) up -d clair
.PHONY: local-dev-up-static
local-dev-up-static: local-dev-clean
$(DOCKER_COMPOSE) -f docker-compose.static up -d redis quay-db
$(DOCKER) exec -it quay-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
DOCKER_USER="$$(id -u):0" $(DOCKER_COMPOSE) -f docker-compose.static up -d --build quay
@echo "You can now access the frontend at http://localhost:8080"
$(DOCKER_COMPOSE) -f docker-compose.static up -d clair-db
$(DOCKER) exec -it clair-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
$(DOCKER_COMPOSE) -f docker-compose.static up -d clair
.PHONY: local-dev-down
local-dev-down:
$(DOCKER_COMPOSE) down
$(MAKE) local-dev-clean
View Git Blame Copy Permalink