psono/psono-server
psono/psono-server
1
0
Fork 0
mirror of https://gitlab.com/psono/psono-server synced 2025-04-18 09:44:01 +03:00
Code
psono-server/.gitlab-ci.yml
Sascha Pfeiffer 596e307b1d Minor adjustment 
Signed-off-by: Sascha Pfeiffer <sascha.pfeiffer@esaqa.com>
2025-04-16 12:20:21 +02:00

435 lines
18 KiB
YAML
Raw Permalink Blame History

variables:
CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
CONTAINER_TEST_ARTIFACTORY_IMAGE: psono-docker.jfrog.io/psono/psono-server:$CI_COMMIT_REF_NAME
CONTAINER_LATEST_IMAGE: psono-docker.jfrog.io/psono/psono-server:latest
stages:
- translate
- build
- test
- release
- deploy
upload-translations:
except:
- schedules
stage: translate
image: psono-docker.jfrog.io/ubuntu:22.04
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- apt-get update && apt-get install -y curl iptables libdevmapper1.02.1 python3 python3-pip
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/containerd.io_1.6.28-1_amd64.deb" -o containerd.deb && echo "3fcc3e44e6d507c346ae05bbaef8f4bb04dfa8da9f04af658537f7e373c91340 containerd.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-buildx-plugin_0.12.1-1~ubuntu.22.04~jammy_amd64.deb" -o docker-buildx-plugin.deb && echo "3fb7cd11fd9bf7c31d81c1afa96c023519d9e6e17422192aa4d0aa2767ed0776 docker-buildx-plugin.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce-cli_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce-cli.deb && echo "624abe860a3d9dd190c278263b3ae115d54ce1135774a79f84026a6ec6ebafd1 docker-ce-cli.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce.deb && echo "63c7703814558a829ab3cc37c8018c050c9dad56f99c261090500f6c87821fd1 docker-ce.deb" | sha256sum -c -
- dpkg -i ./containerd.deb ./docker-ce.deb ./docker-ce-cli.deb ./docker-buildx-plugin.deb
- docker info
- docker build -f DockerfileUbuntu -t ubu-testimage --pull .
- mkdir locale
- docker run -v $(pwd)/locale:/root/psono/locale ubu-testimage bash -c "cd /root/psono && django-admin makemessages --locale=en -i 'restapi' -i 'fileserver' -i 'administration'"
- pip3 install requests
- python3 var/upload_translations.py
only:
- develop@psono/psono-server
create-translations:
except:
- schedules
stage: translate
image: psono-docker.jfrog.io/ubuntu:22.04
script:
- apt-get update && apt-get install -y python3 python3-pip
- pip3 install requests openai==1.55.3
- python3 var/translate.py
only:
- master@psono/psono-server
build-container-alpine:
except:
- schedules
stage: build
image: psono-docker.jfrog.io/ubuntu:22.04
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- sh ./var/update_version.sh
- apt-get update || true
- apt install -y curl ca-certificates iptables libdevmapper1.02.1 python3 python3-pip
- pip3 install requests
- python3 var/download_translations.py
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/containerd.io_1.6.28-1_amd64.deb" -o containerd.deb && echo "3fcc3e44e6d507c346ae05bbaef8f4bb04dfa8da9f04af658537f7e373c91340 containerd.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-buildx-plugin_0.12.1-1~ubuntu.22.04~jammy_amd64.deb" -o docker-buildx-plugin.deb && echo "3fb7cd11fd9bf7c31d81c1afa96c023519d9e6e17422192aa4d0aa2767ed0776 docker-buildx-plugin.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce-cli_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce-cli.deb && echo "624abe860a3d9dd190c278263b3ae115d54ce1135774a79f84026a6ec6ebafd1 docker-ce-cli.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce.deb && echo "63c7703814558a829ab3cc37c8018c050c9dad56f99c261090500f6c87821fd1 docker-ce.deb" | sha256sum -c -
- dpkg -i ./containerd.deb ./docker-ce.deb ./docker-ce-cli.deb ./docker-buildx-plugin.deb
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- docker context create builder
- docker buildx create builder --use
- docker buildx build -f DockerfileAlpine --platform linux/amd64,linux/arm64 -t $CONTAINER_TEST_IMAGE --push --pull .
only:
- branches@psono/psono-server
- tags@psono/psono-server
run-unittests-ubuntu:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:25.0.3-git
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
PSONO_EMAIL_HOST: 172.17.0.1
PSONO_EMAIL_FROM: test@example.com
PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
script:
- docker info
- sh ./var/update_version.sh
- docker build -f DockerfileUbuntu -t ubu-testimage --pull .
- docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust --name db postgres:15.5-alpine
- sleep 20
- docker run --link db:postgres -e "PSONO_THROTTLING=False" ubu-testimage bash -c "apt-get update && apt-get install -y python3-pip && pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && coverage3 run --source='.' ./psono/manage.py test restapi.tests administration.tests fileserver.tests && coverage3 report --omit=psono/restapi/migrations/*,psono/administration/tests*,psono/administration/migrations/*,psono/restapi/tests*,psono/fileserver/migrations/*,psono/fileserver/tests*"
run-unittests-alpine:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:25.0.3-git
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
PSONO_EMAIL_HOST: 172.17.0.1
PSONO_EMAIL_FROM: test@example.com
PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
script:
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- docker pull $CONTAINER_TEST_IMAGE
- docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust --name db postgres:15.5-alpine
- sleep 20
- docker run --link db:postgres -e "PSONO_THROTTLING=False" $CONTAINER_TEST_IMAGE /bin/sh -c "pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && python3 ./psono/manage.py test --parallel=8 restapi.tests administration.tests fileserver.tests"
only:
- branches@psono/psono-server
- tags@psono/psono-server
run-unittests-alpine-arm64:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:25.0.3-git
tags:
- arm64
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
PSONO_EMAIL_HOST: 172.17.0.1
PSONO_EMAIL_FROM: test@example.com
PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
script:
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- docker pull $CONTAINER_TEST_IMAGE
- docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust --name db postgres:15.5-alpine
- sleep 20
- docker run --link db:postgres -e "PSONO_THROTTLING=False" $CONTAINER_TEST_IMAGE /bin/sh -c "pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && python3 ./psono/manage.py test --parallel=8 restapi.tests administration.tests fileserver.tests"
only:
- branches@psono/psono-server
- tags@psono/psono-server
run-vulnerability-python-dependencies:
except:
- schedules
stage: test
image: python:3.11-buster
script:
- pip3 install pip-audit
- pip-audit -r ./requirements.txt
allow_failure: true
scan-docker-image-trivy:
stage: test
except:
- schedules
image: psono-docker.jfrog.io/ubuntu:22.04
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- apt-get update || true
- apt install -y curl ca-certificates iptables libdevmapper1.02.1
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/containerd.io_1.6.28-1_amd64.deb" -o containerd.deb && echo "3fcc3e44e6d507c346ae05bbaef8f4bb04dfa8da9f04af658537f7e373c91340 containerd.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-buildx-plugin_0.12.1-1~ubuntu.22.04~jammy_amd64.deb" -o docker-buildx-plugin.deb && echo "3fb7cd11fd9bf7c31d81c1afa96c023519d9e6e17422192aa4d0aa2767ed0776 docker-buildx-plugin.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce-cli_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce-cli.deb && echo "624abe860a3d9dd190c278263b3ae115d54ce1135774a79f84026a6ec6ebafd1 docker-ce-cli.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce.deb && echo "63c7703814558a829ab3cc37c8018c050c9dad56f99c261090500f6c87821fd1 docker-ce.deb" | sha256sum -c -
- dpkg -i ./containerd.deb ./docker-ce.deb ./docker-ce-cli.deb ./docker-buildx-plugin.deb
- docker info
- docker pull $CONTAINER_TEST_IMAGE
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- curl -fSL "https://github.com/aquasecurity/trivy/releases/download/v0.56.2/trivy_0.56.2_Linux-64bit.deb" -o trivy.deb && echo "c8586e310a195ed1927a275538b45a28d605f41a84d247942198f2d29e7ed310 trivy.deb" | sha256sum -c -
- dpkg -i ./trivy.deb
- trivy --version
- time trivy clean --scan-cache
- trivy image --cache-dir .trivycache/ --severity HIGH,CRITICAL $CONTAINER_TEST_IMAGE
- time trivy image --exit-code 0 --cache-dir .trivycache/ --no-progress "$CONTAINER_TEST_IMAGE"
- time trivy image --exit-code 1 --cache-dir .trivycache/ --severity CRITICAL --no-progress "$CONTAINER_TEST_IMAGE"
cache:
paths:
- .trivycache/
only:
- develop@psono/psono-server
allow_failure: true
run-vulnerability-scan:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:25.0.3-git
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
# - echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- docker pull $CONTAINER_TEST_IMAGE
- docker run -e "LANG=C.UTF-8" $CONTAINER_TEST_IMAGE sh -c "pip3 install bandit && bandit -r /root -x /root/examples,/root/psono/restapi/tests,/root/psono/administration/tests,/root/psono/fileserver/tests"
only:
- branches@psono/psono-server
- tags@psono/psono-server
allow_failure: true
#deploy-security-scan-image:
# except:
# - schedules
# stage: deploy
# image: psono-docker.jfrog.io/docker:25.0.3-git
# services:
# - name: docker:25-dind
# alias: docker
# command: ["--tls=false"]
# variables:
# DOCKER_DRIVER: overlay2
# DOCKER_HOST: tcp://docker:2375/
# DOCKER_TLS_CERTDIR: ""
# script:
# - docker info
# - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
# - echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
# - docker pull $CONTAINER_TEST_IMAGE
# - docker tag $CONTAINER_TEST_IMAGE psono/security-scans:psono-server-ce-$CI_COMMIT_REF_NAME
# - docker push psono/security-scans:psono-server-ce-$CI_COMMIT_REF_NAME
build-sbom:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/ubuntu:22.04
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- apt-get update || true
- apt install -y curl ca-certificates iptables libdevmapper1.02.1 python3 python3-pip
- pip3 install requests
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/containerd.io_1.6.28-1_amd64.deb" -o containerd.deb && echo "3fcc3e44e6d507c346ae05bbaef8f4bb04dfa8da9f04af658537f7e373c91340 containerd.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-buildx-plugin_0.12.1-1~ubuntu.22.04~jammy_amd64.deb" -o docker-buildx-plugin.deb && echo "3fb7cd11fd9bf7c31d81c1afa96c023519d9e6e17422192aa4d0aa2767ed0776 docker-buildx-plugin.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce-cli_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce-cli.deb && echo "624abe860a3d9dd190c278263b3ae115d54ce1135774a79f84026a6ec6ebafd1 docker-ce-cli.deb" | sha256sum -c -
- curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce.deb && echo "63c7703814558a829ab3cc37c8018c050c9dad56f99c261090500f6c87821fd1 docker-ce.deb" | sha256sum -c -
- dpkg -i ./containerd.deb ./docker-ce.deb ./docker-ce-cli.deb ./docker-buildx-plugin.deb
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- docker pull $CONTAINER_TEST_IMAGE
- curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
- syft scan $CONTAINER_TEST_IMAGE -o cyclonedx-json > sbom.json
- mv ./sbom.json ../
- rm -Rf *
- rm -Rf .* 2> /dev/null || true
- mv ../sbom.json ./
artifacts:
name: "sbom_$CI_COMMIT_REF_NAME"
paths:
- ./*
only:
- branches@psono/psono-server
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
release-container:
except:
- schedules
stage: release
image: psono-docker.jfrog.io/docker:25.0.3-git
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- apk add skopeo
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- skopeo copy --all docker://$CONTAINER_TEST_IMAGE docker://$CONTAINER_LATEST_IMAGE
- skopeo copy --all docker://$CONTAINER_TEST_IMAGE docker://$CONTAINER_TEST_ARTIFACTORY_IMAGE
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
release-version-bump:
except:
- schedules
stage: release
image: psono-docker.jfrog.io/docker:25.0.3-git
script:
- mkdir -p /root/.ssh
- echo "$github_deploy_key" > /root/.ssh/id_rsa
- chmod 600 /root/.ssh/id_rsa
- ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts
- git remote set-url --push origin "git@gitlab.com:${CI_PROJECT_PATH}.git"
- git config --global user.email "buildserver@psono.com"
- git config --global user.name "Build Server"
- git checkout master
- git pull
- sh ./var/update_version.sh
- git add psono/VERSION.txt
- git add psono/SHA.txt
- git commit -m "Bump version [skip ci]"
- git push
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
deploy:
except:
- schedules
stage: deploy
image: psono-docker.jfrog.io/docker:25.0.3-git
services:
- name: docker:25-dind
alias: docker
command: ["--tls=false"]
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
DOCKER_TLS_CERTDIR: ""
script:
- docker info
- echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- sh ./var/deploy.sh
environment:
name: production
url: https://psono.pw
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
deploy-changelog:
except:
- schedules
stage: deploy
image: psono-docker.jfrog.io/ubuntu:22.04
script:
- sh ./var/deploy_changelog.sh
environment:
name: static.psono.com
url: https://static.psono.com/gitlab.com/psono/psono-server/changelog.json
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
deploy-nightlyartifacts:
except:
- schedules
stage: release
image: psono-docker.jfrog.io/ubuntu:22.04
dependencies:
- build-sbom
script:
- sh ./var/deploy_nightlyartifacts.sh
environment:
name: static.psono.com
url: https://get.psono.com/psono/psono-server/nightly/sbom.json
only:
- develop@psono/psono-server
deploy-releaseartifacts:
except:
- schedules
stage: deploy
image: psono-docker.jfrog.io/ubuntu:22.04
dependencies:
- build-sbom
script:
- sh ./var/deploy_releaseartifacts.sh
environment:
name: static.psono.com
url: https://get.psono.com/psono/psono-server/latest/sbom.json
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
View Git Blame Copy Permalink