1
0
mirror of https://gitlab.com/psono/psono-fileserver synced 2025-04-18 12:24:05 +03:00

Initial commit

This commit is contained in:
Sascha Pfeiffer 2018-12-21 21:57:53 +01:00
commit 60e8166002
67 changed files with 2567 additions and 0 deletions

41
.codeclimate.yml Normal file
View File

@ -0,0 +1,41 @@
---
engines:
csslint:
enabled: true
checks:
box-sizing:
enabled: false
duplication:
enabled: false
checks:
Identical code:
enabled: false
config:
languages:
python:
mass_threshold: 190
eslint:
enabled: true
fixme:
enabled: true
checks:
TODO:
enabled: false
radon:
enabled: true
ratings:
paths:
- "**.css"
- "**.inc"
- "**.js"
- "**.jsx"
- "**.module"
- "**.php"
- "**.py"
- "**.rb"
exclude_paths:
- configs/**/*
- var/**/*
- psono/restapi/tests/**/*
- psono/restapi/migrations/**/*
- psono/static/**/*

2
.csslintrc Normal file
View File

@ -0,0 +1,2 @@
--exclude-exts=.min.css
--ignore=adjoining-classes,box-model,ids,order-alphabetical,unqualified-attributes

21
.dockerignore Normal file
View File

@ -0,0 +1,21 @@
configs/apache
configs/nginx
demo
.coverage
.htmlcov
.docs
.dockerignore
.git
.docu
docu
.codeclimate.yml
.csslintrc
.eslintignore
.eslintrc.yml
.gitignore
.gitlab-ci.yml
.gitmodules
Dockerfile*
README.md

1
.eslintignore Normal file
View File

@ -0,0 +1 @@
**/*{.,-}min.js

277
.eslintrc.yml Normal file
View File

@ -0,0 +1,277 @@
---
parserOptions:
sourceType: module
ecmaFeatures:
jsx: true
env:
amd: true
browser: true
es6: true
jquery: true
node: true
# http://eslint.org/docs/rules/
rules:
# Possible Errors
no-await-in-loop: off
no-cond-assign: error
no-console: off
no-constant-condition: error
no-control-regex: error
no-debugger: error
no-dupe-args: error
no-dupe-keys: error
no-duplicate-case: error
no-empty-character-class: error
no-empty: error
no-ex-assign: error
no-extra-boolean-cast: error
no-extra-parens: off
no-extra-semi: error
no-func-assign: error
no-inner-declarations:
- error
- functions
no-invalid-regexp: error
no-irregular-whitespace: error
no-negated-in-lhs: error
no-obj-calls: error
no-prototype-builtins: off
no-regex-spaces: error
no-sparse-arrays: error
no-template-curly-in-string: off
no-unexpected-multiline: error
no-unreachable: error
no-unsafe-finally: off
no-unsafe-negation: off
use-isnan: error
valid-jsdoc: off
valid-typeof: error
# Best Practices
accessor-pairs: error
array-callback-return: off
block-scoped-var: off
class-methods-use-this: off
complexity:
- error
- 6
consistent-return: off
curly: off
default-case: off
dot-location: off
dot-notation: off
eqeqeq: error
guard-for-in: error
no-alert: error
no-caller: error
no-case-declarations: error
no-div-regex: error
no-else-return: off
no-empty-function: off
no-empty-pattern: error
no-eq-null: error
no-eval: error
no-extend-native: error
no-extra-bind: error
no-extra-label: off
no-fallthrough: error
no-floating-decimal: off
no-global-assign: off
no-implicit-coercion: off
no-implied-eval: error
no-invalid-this: off
no-iterator: error
no-labels:
- error
- allowLoop: true
allowSwitch: true
no-lone-blocks: error
no-loop-func: error
no-magic-number: off
no-multi-spaces: off
no-multi-str: off
no-native-reassign: error
no-new-func: error
no-new-wrappers: error
no-new: error
no-octal-escape: error
no-octal: error
no-param-reassign: off
no-proto: error
no-redeclare: error
no-restricted-properties: off
no-return-assign: error
no-return-await: off
no-script-url: error
no-self-assign: off
no-self-compare: error
no-sequences: off
no-throw-literal: off
no-unmodified-loop-condition: off
no-unused-expressions: error
no-unused-labels: off
no-useless-call: error
no-useless-concat: error
no-useless-escape: off
no-useless-return: off
no-void: error
no-warning-comments: off
no-with: error
prefer-promise-reject-errors: off
radix: error
require-await: off
vars-on-top: off
wrap-iife: error
yoda: off
# Strict
strict: off
# Variables
init-declarations: off
no-catch-shadow: error
no-delete-var: error
no-label-var: error
no-restricted-globals: off
no-shadow-restricted-names: error
no-shadow: off
no-undef-init: error
no-undef: off
no-undefined: off
no-unused-vars: off
no-use-before-define: off
# Node.js and CommonJS
callback-return: error
global-require: error
handle-callback-err: error
no-mixed-requires: off
no-new-require: off
no-path-concat: error
no-process-env: off
no-process-exit: error
no-restricted-modules: off
no-sync: off
# Stylistic Issues
array-bracket-spacing: off
block-spacing: off
brace-style: off
camelcase: off
capitalized-comments: off
comma-dangle:
- error
- never
comma-spacing: off
comma-style: off
computed-property-spacing: off
consistent-this: off
eol-last: off
func-call-spacing: off
func-name-matching: off
func-names: off
func-style: off
id-length: off
id-match: off
indent: off
jsx-quotes: off
key-spacing: off
keyword-spacing: off
line-comment-position: off
linebreak-style: off
lines-around-comment: off
lines-around-directive: off
max-depth: off
max-len: off
max-nested-callbacks: off
max-params: off
max-statements-per-line: off
max-statements:
- error
- 30
multiline-ternary: off
new-cap: off
new-parens: off
newline-after-var: off
newline-before-return: off
newline-per-chained-call: off
no-array-constructor: off
no-bitwise: off
no-continue: off
no-inline-comments: off
no-lonely-if: off
no-mixed-operators: off
no-mixed-spaces-and-tabs: off
no-multi-assign: off
no-multiple-empty-lines: off
no-negated-condition: off
no-nested-ternary: off
no-new-object: off
no-plusplus: off
no-restricted-syntax: off
no-spaced-func: off
no-tabs: off
no-ternary: off
no-trailing-spaces: off
no-underscore-dangle: off
no-unneeded-ternary: off
object-curly-newline: off
object-curly-spacing: off
object-property-newline: off
one-var-declaration-per-line: off
one-var: off
operator-assignment: off
operator-linebreak: off
padded-blocks: off
quote-props: off
quotes: off
require-jsdoc: off
semi-spacing: off
semi: off
sort-keys: off
sort-vars: off
space-before-blocks: off
space-before-function-paren: off
space-in-parens: off
space-infix-ops: off
space-unary-ops: off
spaced-comment: off
template-tag-spacing: off
unicode-bom: off
wrap-regex: off
# ECMAScript 6
arrow-body-style: off
arrow-parens: off
arrow-spacing: off
constructor-super: off
generator-star-spacing: off
no-class-assign: off
no-confusing-arrow: off
no-const-assign: off
no-dupe-class-members: off
no-duplicate-imports: off
no-new-symbol: off
no-restricted-imports: off
no-this-before-super: off
no-useless-computed-key: off
no-useless-constructor: off
no-useless-rename: off
no-var: off
object-shorthand: off
prefer-arrow-callback: off
prefer-const: off
prefer-destructuring: off
prefer-numeric-literals: off
prefer-rest-params: off
prefer-reflect: off
prefer-spread: off
prefer-template: off
require-yield: off
rest-spread-spacing: off
sort-imports: off
symbol-description: off
template-curly-spacing: off
yield-star-spacing: off

11
.gitignore vendored Normal file
View File

@ -0,0 +1,11 @@
.idea
.mypy_cache
*.py[cod]
problemtest
.coverage*
log
docs/
htmlcov/
test/
psono/static/admin
psono/static/rest_framework

198
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,198 @@
variables:
CONTAINER_TEST_IMAGE: psono-docker.jfrog.io/psono/psono-fileserver:$CI_BUILD_REF_NAME
CONTAINER_LATEST_IMAGE: psono-docker.jfrog.io/psono/psono-fileserver:latest
stages:
- build
- test
- release
- deploy
build-container-alpine:
except:
- schedules
stage: build
image: psono-docker.jfrog.io/ubuntu:16.04
services:
- docker:dind
variables:
DOCKER_HOST: 'tcp://docker:2375'
script:
- sh ./var/update_version.sh
- apt-get update && apt-get install -y curl
- curl -fSL "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz" -o docker.tgz && echo "1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 *docker.tgz" | sha256sum -c - && tar -xzvf docker.tgz && mv docker/* /usr/local/bin/ && rm -Rf docker*
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- docker build -f DockerfileAlpine -t $CONTAINER_TEST_IMAGE --pull .
- docker push $CONTAINER_TEST_IMAGE
- curl -fL https://getcli.jfrog.io | sh
- ./jfrog rt c rt-server-1 --url=https://psono.jfrog.io/psono --user=gitlab --password=$artifactory_credentials
- ./jfrog rt sp "docker/psono/psono-fileserver/$CI_BUILD_REF_NAME/manifest.json" "CI_BUILD_REF_NAME=$CI_BUILD_REF_NAME;CI_COMMIT_SHA=$CI_COMMIT_SHA;CI_COMMIT_URL=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA;CI_PROJECT_ID=$CI_PROJECT_ID;CI_PROJECT_NAME=$CI_PROJECT_NAME;CI_PROJECT_NAMESPACE=$CI_PROJECT_NAMESPACE;CI_PROJECT_URL=$CI_PROJECT_URL;CI_PIPELINE_ID=$CI_PIPELINE_ID;CI_PIPELINE_URL=$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID;CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME;CI_JOB_ID=$CI_JOB_ID;CI_JOB_URL=$CI_PROJECT_URL/-/jobs/$CI_JOB_ID;CI_JOB_NAME=$CI_JOB_NAME;CI_JOB_STAGE=$CI_JOB_STAGE;CI_RUNNER_ID=$CI_RUNNER_ID;GITLAB_USER_ID=$GITLAB_USER_ID;CI_SERVER_VERSION=$CI_SERVER_VERSION"
- ./jfrog rt sp "docker/psono/psono-fileserver/$CI_BUILD_REF_NAME/manifest.json" "CI_COMMIT_TAG=$CI_COMMIT_TAG" || true
run-unittests-ubuntu1604:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:git
variables:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
PSONO_EMAIL_HOST: 172.17.0.1
PSONO_EMAIL_FROM: test@example.com
PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
services:
- docker:dind
script:
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- sh ./var/update_version.sh
- docker build -f DockerfileUbuntu1604 -t ubu1604-testimage --pull .
- docker run -d --name db postgres:9.6
- sleep 20
- docker run --link db:postgres ubu1604-testimage bash -c "apt-get update && apt-get install -y python3-pip && pip3 install -r requirements-dev.txt && pip3 install mypy && python3 /usr/local/bin/mypy -p psono --ignore-missing-imports && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && coverage3 run --source='.' ./psono/manage.py test restapi.tests cron.tests && coverage3 report --omit=psono/restapi/migrations/*,psono/administration/tests*,psono/administration/migrations/*,psono/restapi/tests*"
run-unittests-alpine:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:git
variables:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
PSONO_EMAIL_HOST: 172.17.0.1
PSONO_EMAIL_FROM: test@example.com
PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
services:
- docker:dind
script:
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- docker pull $CONTAINER_TEST_IMAGE
- docker run -d --name db postgres:9.6
- sleep 20
- docker run --link db:postgres $CONTAINER_TEST_IMAGE /bin/sh -c "pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && python3 ./psono/manage.py test --parallel=8 restapi.tests cron.tests"
run-unittests-centos7:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:git
variables:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: ""
PSONO_EMAIL_HOST: 172.17.0.1
PSONO_EMAIL_FROM: test@example.com
PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
services:
- docker:dind
script:
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- sh ./var/update_version.sh
- docker build -f DockerfileCentos7 -t centos7-testimage --pull .
- docker run -d --name db postgres:9.6
- sleep 20
- docker run --link db:postgres centos7-testimage bash -c "yum -y install python34-pip && pip3 install -r requirements-dev.txt && pip3 install mypy && python3 /usr/bin/mypy -p psono --ignore-missing-imports && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && python3 ./psono/manage.py test --parallel=8 restapi.tests cron.tests"
run-vulnerability-scan:
except:
- schedules
stage: test
image: psono-docker.jfrog.io/docker:git
services:
- docker:dind
script:
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- docker pull $CONTAINER_TEST_IMAGE
- docker run -e "LANG=C.UTF-8" $CONTAINER_TEST_IMAGE sh -c "pip3 install safety && safety check"
- docker run -e "LANG=C.UTF-8" $CONTAINER_TEST_IMAGE sh -c "pip3 install bandit && bandit -r /root -x /root/psono/restapi/tests,/root/psono/administration/tests"
#deploy-security-scan-image:
# except:
# - schedules
# stage: deploy
# image: psono-docker.jfrog.io/docker:git
# services:
# - docker:dind
# script:
# - docker info
# - echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
# - echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
# - echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
# - docker pull $CONTAINER_TEST_IMAGE
# - docker tag $CONTAINER_TEST_IMAGE psono/security-scans:psono-fileserver-ce-$CI_BUILD_REF_NAME
# - docker push psono/security-scans:psono-fileserver-ce-$CI_BUILD_REF_NAME
release-container:
except:
- schedules
stage: release
image: psono-docker.jfrog.io/docker:git
services:
- docker:dind
script:
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- docker pull $CONTAINER_TEST_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_LATEST_IMAGE
- docker push $CONTAINER_LATEST_IMAGE
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
deploy:
except:
- schedules
stage: deploy
image: psono-docker.jfrog.io/docker:git
services:
- docker:dind
script:
- docker info
- echo $CI_BUILD_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
- echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
- echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
- sh ./var/deploy.sh
environment:
name: production
url: https://psono.pw
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/
deploy-changelog:
except:
- schedules
stage: deploy
image: psono-docker.jfrog.io/ubuntu:16.04
script:
- sh ./var/deploy_changelog.sh
environment:
name: static.psono.com
url: https://static.psono.com/gitlab.com/psono/psono-fileserver/changelog.json
only:
- /^v[0-9]*\.[0-9]*\.[0-9]*$/

1
.gitmodules vendored Normal file
View File

@ -0,0 +1 @@

47
DockerfileAlpine Normal file
View File

@ -0,0 +1,47 @@
# PSONO Dockerfile for Alpine
FROM psono-docker.jfrog.io/python:alpine3.7
LABEL maintainer="Sascha Pfeiffer <sascha.pfeiffer@psono.com>"
COPY psono/static/email /var/www/html/static/email
COPY . /root/
WORKDIR /root
RUN apk upgrade --no-cache && \
mkdir -p /root/.pip && \
echo '[global]' >> /root/.pip/pip.conf && \
echo 'index-url = https://psono.jfrog.io/psono/api/pypi/pypi/simple' >> /root/.pip/pip.conf && \
apk add --no-cache \
dcron \
curl \
build-base \
libffi-dev \
linux-headers \
pip3 install -r requirements.txt && \
pip3 install uwsgi && \
mkdir -p /root/.psono_fileserver /var/log/cron && \
echo "* * * * * ( sleep 5; touch /tmp/psono_fileserver_ping && curl -f http://localhost/cron/ping/ && touch /tmp/psono_fileserver_ping_success )" >> /etc/crontabs/root && \
echo "* * * * * ( sleep 15; touch /tmp/psono_fileserver_ping && curl -f http://localhost/cron/ping/ && touch /tmp/psono_fileserver_ping_success )" >> /etc/crontabs/root && \
echo "* * * * * ( sleep 25; touch /tmp/psono_fileserver_ping && curl -f http://localhost/cron/ping/ && touch /tmp/psono_fileserver_ping_success )" >> /etc/crontabs/root && \
echo "* * * * * ( sleep 35; touch /tmp/psono_fileserver_ping && curl -f http://localhost/cron/ping/ && touch /tmp/psono_fileserver_ping_success )" >> /etc/crontabs/root && \
echo "* * * * * ( sleep 45; touch /tmp/psono_fileserver_ping && curl -f http://localhost/cron/ping/ && touch /tmp/psono_fileserver_ping_success )" >> /etc/crontabs/root && \
echo "* * * * * ( sleep 55; touch /tmp/psono_fileserver_ping && curl -f http://localhost/cron/ping/ && touch /tmp/psono_fileserver_ping_success )" >> /etc/crontabs/root && \
cp /root/configs/mainconfig/settings.yaml /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresDatabase/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresUser/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresHost/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresPort/5432/g /root/.psono_fileserver/settings.yaml && \
sed -i s,path/to/psono-fileserver,root,g /root/.psono_fileserver/settings.yaml && \
apk del --no-cache \
build-base \
libffi-dev \
linux-headers && \
rm -Rf \
/root/.cache
HEALTHCHECK --interval=2m --timeout=3s \
CMD curl -f http://localhost/healthcheck/ || exit 1
EXPOSE 80
CMD ["/bin/sh", "/root/configs/docker/cmd.sh"]

49
DockerfileCentos7 Normal file
View File

@ -0,0 +1,49 @@
# PSONO Dockerfile for CentOS 7
FROM psono-docker.jfrog.io/centos:centos7
LABEL maintainer="Sascha Pfeiffer <sascha.pfeiffer@psono.com>"
COPY psono/static/email /var/www/html/static/email
COPY . /root/
WORKDIR /root
RUN mkdir -p /root/.pip && \
echo '[global]' >> /root/.pip/pip.conf && \
echo 'index-url = https://psono.jfrog.io/psono/api/pypi/pypi/simple' >> /root/.pip/pip.conf && \
yum -y update && \
yum -y install epel-release && \
yum -y update && \
yum -y install \
gcc \
haveged \
libffi-devel \
openssl-devel \
postgresql \
postgresql-devel \
postgresql-client \
python34 \
python34-devel \
python34-pip && \
pip3 install -r requirements.txt && \
pip3 install uwsgi && \
pip3 install typing && \
mkdir -p /root/.psono_fileserver && \
cp /root/configs/mainconfig/settings.yaml /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresDatabase/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresUser/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresHost/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresPort/5432/g /root/.psono_fileserver/settings.yaml && \
sed -i s,path/to/psono-fileserver,root,g /root/.psono_fileserver/settings.yaml && \
yum remove -y \
python34-pip && \
yum clean all && \
rm -Rf \
/root/requirements.txt \
/root/psono/static \
/root/var \
/root/.cache \
/tmp/* \
/var/tmp/*
EXPOSE 80
CMD ["/bin/sh", "/root/configs/docker/cmd.sh"]

41
DockerfileUbuntu1604 Normal file
View File

@ -0,0 +1,41 @@
# PSONO Dockerfile for Ubuntu 16.04
FROM psono-docker.jfrog.io/ubuntu:16.04
ENV DEBIAN_FRONTEND noninteractive
LABEL maintainer="Sascha Pfeiffer <sascha.pfeiffer@psono.com>"
COPY psono/static/email /var/www/html/static/email
COPY . /root/
WORKDIR /root
RUN mkdir -p /root/.pip && \
echo '[global]' >> /root/.pip/pip.conf && \
echo 'index-url = https://psono.jfrog.io/psono/api/pypi/pypi/simple' >> /root/.pip/pip.conf && \
apt-get update && \
apt-get install -y \
haveged \
libyaml-dev \
libpython3-dev \
libpq-dev \
libffi-dev \
python3-dev \
python3-pip \
python3-psycopg2 \
postgresql-client && \
pip3 install -r requirements.txt && \
pip3 install uwsgi && \
mkdir -p /root/.psono_fileserver && \
cp /root/configs/mainconfig/settings.yaml /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresDatabase/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresUser/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresHost/postgres/g /root/.psono_fileserver/settings.yaml && \
sed -i s/YourPostgresPort/5432/g /root/.psono_fileserver/settings.yaml && \
sed -i s,path/to/psono-fileserver,root,g /root/.psono_fileserver/settings.yaml && \
apt-get purge -y \
python3-pip && \
apt-get clean && \
rm -Rf /root/psono/static && \
rm -Rf /root/var && \
rm -Rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache
EXPOSE 80
CMD ["/bin/sh", "/root/configs/docker/cmd.sh"]

178
LICENSE.md Normal file
View File

@ -0,0 +1,178 @@
# License:
Copyright 2017 Sascha Pfeiffer
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
# Licenses of used software
## Django Restframework: BSD 2-clause
Copyright (c) 2011-2015, Tom Christie
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----
## Django: BSD 3-clause
Copyright (c) Django Software Foundation and individual contributors.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Django nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----
## django-rest-auth: MIT
The MIT License (MIT)
Copyright (c) 2014 Tivix
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
----
## django-all-auth: MIT
The MIT License (MIT)
Copyright (c) 2010-2015 Raymond Penners and contributors
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
## ltreefield: MIT
Copyright (c) 2014 whitglint
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
----
## leemunroe/responsive-html-email-template: MIT
The MIT License (MIT)
Copyright (c) [2013] [Lee Munroe]
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

40
README.md Normal file
View File

@ -0,0 +1,40 @@
# PSONO Fileserver - Password Manager
Master: [![build status](https://gitlab.com/psono/psono-fileserver/badges/master/build.svg)](https://gitlab.com/psono/psono-fileserver/commits/master) [![coverage report](https://gitlab.com/psono/psono-fileserver/badges/master/coverage.svg)](https://gitlab.com/psono/psono-fileserver/commits/master) [![Code Climate](https://codeclimate.com/github/psono/psono-fileserver/badges/gpa.svg)](https://codeclimate.com/github/psono/psono-fileserver) [![build status](https://images.microbadger.com/badges/image/psono/psono-fileserver.svg)](https://hub.docker.com/r/psono/psono-fileserver/) [![build status](https://img.shields.io/docker/pulls/psono/psono-fileserver.svg)](https://hub.docker.com/r/psono/psono-fileserver/)
Develop: [![build status](https://gitlab.com/psono/psono-fileserver/badges/develop/build.svg)](https://gitlab.com/psono/psono-fileserver/commits/develop) [![coverage report](https://gitlab.com/psono/psono-fileserver/badges/develop/coverage.svg)](https://gitlab.com/psono/psono-fileserver/commits/develop)
# Canonical source
The canonical source of PSONO Fileserver is [hosted on GitLab.com](https://gitlab.com/psono/psono-fileserver).
# Documentation
The documentation for the psono fileserver can be found here:
[Psono Documentation](https://doc.psono.com/)
Some things that have not yet found their place in the documentation:
## Storage Engines:
Psono Fileserver is using "django-storages" as storage engine. The official documentation for django-storages can be found here:
http://django-storages.readthedocs.io/en/latest/index.html
Supported Provider are:
* Amazon S3
* Apache Libcloud
* Azure Storage
* DropBox
* FTP
* Google Cloud Storage
* SFTP
storages.backends.s3boto3.S3Boto3Storage
## LICENSE
Visit the [License.md](/LICENSE.md) for more details

View File

@ -0,0 +1,61 @@
ServerSignature Off
ServerTokens Prod
SSLStaplingCache shmcb:/var/run/ocsp(128000)
WSGIPythonPath /path/to/psono-fileserver/psono
<VirtualHost *:80>
ServerName dev.psono.pw
ServerSignature Off
RewriteEngine on
RewriteCond %{HTTPS} !=on
RewriteRule .* https://%{SERVER_NAME}%{REQUEST_URI} [NE,R,L]
</VirtualHost>
<virtualhost *:443>
ServerName dev.psono.pw
ServerAdmin webmaster@localhost
Header always add Strict-Transport-Security "max-age=15768000"
Header always append X-Frame-Options DENY
Header set X-Content-Type-Options nosniff
Header set X-XSS-Protection "1; mode=block"
Header always set Referrer-Policy "same-origin"
Header set Content-Security-Policy "default-src 'none'; connect-src 'self'; font-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self' 'unsafe-inline'; object-src 'self'; form-action 'self'"
SSLEngine on
# from https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.4.18&openssl=1.0.2g&hsts=yes&profile=modern
SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1
SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
SSLHonorCipherOrder on
SSLCompression off
SSLSessionTickets off
SSLUseStapling on
SSLStaplingResponderTimeout 5
SSLStaplingReturnResponderErrors off
SSLCertificateKeyFile /certificate_path/to/privkey.pem
SSLCertificateFile /certificate_path/to/certificate.pem
SSLCertificateChainFile /certificate_path/to/certificate_chain.pem
ServerSignature Off
WSGIDaemonProcess dev.psono.pw python-path=/path/to/psono-fileserver/psono
WSGIProcessGroup dev.psono.pw
WSGIScriptAlias / /path/to/psono-fileserver/psono/psono/wsgi.py process-group=dev.psono.pw
WSGIPassAuthorization On
<Directory /path/to/psono-fileserver/psono/psono>
<Files wsgi.py>
Require all granted
</Files>
</Directory>
ErrorLog /path/to/log/error.log
CustomLog /path/to/log/access.log combined
</virtualhost>

3
configs/docker/cmd.sh Normal file
View File

@ -0,0 +1,3 @@
crond -b -L /var/log/cron/cron.log
# tail -f /var/log/cron/cron.log &
python3 /root/psono/manage.py migrate && uwsgi --ini /root/configs/docker/psono_uwsgi_port.ini

View File

@ -0,0 +1,6 @@
[uwsgi]
http-socket = :80
chdir = /root/psono
module = psono.wsgi
master = true
processes = 10

View File

@ -0,0 +1,43 @@
# Generate the following six parameters with:
# ./psono/manage.py fileserver fsgenerateserverkeys CLUSTER_ID
SECRET_KEY: 'SOME SUPER SECRET KEY THAT SHOULD BE RANDOM AND 32 OR MORE DIGITS LONG'
PRIVATE_KEY: '302650c3c82f7111c2e8ceb660d32173cdc8c3d7717f1d4f982aad5234648fcb'
PUBLIC_KEY: '02da2ad857321d701d754a7e60d0a147cdbc400ff4465e1f57bc2d9fbfeddf0b'
CLUSTER: 'CLUSTER_ID'
SHARDS: []
# Switch DEBUG to false if you go into production
DEBUG: True
# Adjust this according to Django Documentation https://docs.djangoproject.com/en/2.0/ref/settings/#allowed-hosts
ALLOWED_HOSTS: ['*']
# Should be the URL of the host under which the server is reachable
# If you open the url you should have a text similar to {"detail":"Authentication credentials were not provided."}
HOST_URL: 'https://example.com'
# Should be the URL of the host under which the fileserver is reachable
# If you open the url you should have a text similar to {"detail":"Authentication credentials were not provided."}
FILESERVER_URL: 'https://example.com'
# Cache enabled without belows Redis may lead to unexpected behaviour
CACHE_ENABLE: False
# Cache with Redis
# By deault you should use something different than database 0 or 1, e.g. 13 (default max is 16, can be configured in
# redis.conf) possible URLS are:
# redis://[:password]@localhost:6379/0
# rediss://[:password]@localhost:6379/0
# unix://[:password]@/path/to/socket.sock?db=0
CACHE_REDIS: False
CACHE_REDIS_LOCATION: 'redis://127.0.0.1:6379/13'
# Disables Throttling (necessary for unittests to pass) by overriding the cache with a dummy cache
# https://docs.djangoproject.com/en/2.0/topics/cache/#dummy-caching-for-development
THROTTLING: False
# Disables protections (necessary for unittests to pass)
REPLAY_PROTECTION_DISABLED: True

View File

@ -0,0 +1,46 @@
upstream django {
server unix:///tmp/psono.sock;
}
server {
listen 80;
server_name dev.psono.pw;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
server_name dev.psono.pw;
# from https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.10.0&openssl=1.0.2g&hsts=yes&profile=modern
ssl_protocols TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
ssl_stapling on;
ssl_stapling_verify on;
ssl_session_timeout 1d;
resolver 8.8.8.8 8.8.4.4 valid=300s;
resolver_timeout 5s;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
# Enable the following line only if you know what you are doing :)
# add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload";
add_header Referrer-Policy same-origin;
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; font-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self' 'unsafe-inline'; object-src 'self'; form-action 'self'";
ssl_certificate /path/to/fullchain.pem;
ssl_certificate_key /path/to/privkey.pem;
location / {
uwsgi_pass django;
include /path/to/psono-fileserver/configs/nginx/uwsgi_params;
}
}

View File

@ -0,0 +1,20 @@
[uwsgi]
# Django-related settings
# the base directory (full path)
chdir = /root/psono
# Django's wsgi file
module = psono.wsgi
# the virtualenv (full path)
# home = /path/to/virtualenv
# process-related settings
# master
master = true
# maximum number of worker processes
processes = 10
# the socket (use the full path to be safe
socket = /tmp/psono.sock
# ... with appropriate permissions - may be needed
chmod-socket = 666
# clear environment on exit
vacuum = true

View File

@ -0,0 +1,16 @@
uwsgi_param QUERY_STRING $query_string;
uwsgi_param REQUEST_METHOD $request_method;
uwsgi_param CONTENT_TYPE $content_type;
uwsgi_param CONTENT_LENGTH $content_length;
uwsgi_param REQUEST_URI $request_uri;
uwsgi_param PATH_INFO $document_uri;
uwsgi_param DOCUMENT_ROOT $document_root;
uwsgi_param SERVER_PROTOCOL $server_protocol;
uwsgi_param REQUEST_SCHEME $scheme;
uwsgi_param HTTPS $https if_not_empty;
uwsgi_param REMOTE_ADDR $remote_addr;
uwsgi_param REMOTE_PORT $remote_port;
uwsgi_param SERVER_PORT $server_port;
uwsgi_param SERVER_NAME $server_name;

0
psono/VERSION.txt Normal file
View File

0
psono/__init__.py Normal file
View File

0
psono/cron/__init__.py Normal file
View File

3
psono/cron/admin.py Normal file
View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

5
psono/cron/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class CronConfig(AppConfig):
name = 'cron'

View File

3
psono/cron/models.py Normal file
View File

@ -0,0 +1,3 @@
from django.db import models
# Create your models here.

View File

@ -0,0 +1,14 @@
from rest_framework.permissions import BasePermission
import ipaddress
class AllowLocalhost(BasePermission):
"""
Allow any access from localhost.
"""
def has_permission(self, request, view):
ip = request.META.get('REMOTE_ADDR')
return ipaddress.ip_address(ip).is_loopback

View File

24
psono/cron/urls.py Normal file
View File

@ -0,0 +1,24 @@
"""psono URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Add an import: from blog import urls as blog_urls
2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls))
"""
from django.conf.urls import url
from django.conf import settings
from os.path import join, dirname, abspath
import django
from . import views
urlpatterns = [
url(r'^ping/$', views.PingView.as_view(), name='ping'),
]

View File

@ -0,0 +1 @@
from .ping import PingView

45
psono/cron/views/ping.py Normal file
View File

@ -0,0 +1,45 @@
from rest_framework import status
from rest_framework.response import Response
from rest_framework.generics import GenericAPIView
from ..permission_classes import AllowLocalhost
from restapi.utils import APIServer
class PingView(GenericAPIView):
permission_classes = (AllowLocalhost,)
allowed_methods = ('GET', 'OPTIONS', 'HEAD')
throttle_scope = 'cron'
def get(self, request, *args, **kwargs):
"""
Sends the health status of the file server to the server.
:param request:
:type request:
:param args:
:type args:
:param kwargs:
:type kwargs:
:return:
:rtype:
"""
r = APIServer.query(
endpoint="/fileserver/alive/"
)
if status.is_success(r.status_code):
return Response({}, status=status.HTTP_200_OK)
else:
return Response({}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def put(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def post(self, request, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def delete(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)

15
psono/manage.py Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "psono.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)

0
psono/psono/__init__.py Normal file
View File

369
psono/psono/settings.py Normal file
View File

@ -0,0 +1,369 @@
"""
Django settings for psono project.
Generated by 'django-admin startproject' using Django 2.0.2.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.0/ref/settings/
"""
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import yaml
import json
import nacl.encoding
import nacl.signing
import binascii
import six
import uuid
from corsheaders.defaults import default_headers
import nacl.encoding
import nacl.utils
import nacl.secret
from nacl.public import PrivateKey, PublicKey, Box
from django.conf import global_settings
try:
# Fall back to psycopg2cffi
from psycopg2cffi import compat
compat.register()
except ImportError:
import psycopg2
HOME = os.path.expanduser('~')
with open(os.path.join(HOME, '.psono_fileserver', 'settings.yaml'), 'r') as stream:
config = yaml.safe_load(stream)
def config_get(key, *args):
if 'PSONOFS_' + key in os.environ:
val = os.environ.get('PSONOFS_' + key)
try:
json_object = json.loads(val)
except ValueError:
return val
return json_object
if key in config:
return config.get(key)
if len(args) > 0:
return args[0]
raise Exception("Setting missing", "Couldn't find the setting for %s (maybe you forgot the 'PSONOFS_' prefix in the environment variable" % (key,))
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config_get('SECRET_KEY')
PRIVATE_KEY = config_get('PRIVATE_KEY', '')
PUBLIC_KEY = config_get('PUBLIC_KEY', '')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = config_get('ALLOWED_HOSTS')
READ = config_get('READ', True)
WRITE = config_get('WRITE', True)
IP_READ_WHITELIST = config_get('IP_READ_WHITELIST', [])
IP_WRITE_WHITELIST = config_get('IP_WRITE_WHITELIST', [])
IP_READ_BLACKLIST = config_get('IP_READ_BLACKLIST', [])
IP_WRITE_BLACKLIST = config_get('IP_WRITE_BLACKLIST', [])
CLUSTER_ID = config_get('CLUSTER_ID')
CLUSTER_PRIVATE_KEY = config_get('CLUSTER_PRIVATE_KEY')
SHARDS_PUBLIC = []
SHARDS_DICT = {}
for s in config_get('SHARDS'):
SHARDS_DICT[s['shard_id']] = s
SHARDS_PUBLIC.append({
'shard_id': s['shard_id'],
'read': s['read'] and READ,
'write': s['write'] and WRITE
})
HOST_URL = config_get('HOST_URL')
SERVER_URL = config_get('SERVER_URL')
SERVER_PUBLIC_KEY = config_get('SERVER_PUBLIC_KEY')
SERVER_URL_VERIFY_SSL = config_get('SERVER_URL_VERIFY_SSL', True)
FILESERVER_ID = str(uuid.uuid4())
FILESERVER_SESSION_KEY = nacl.encoding.HexEncoder.encode(nacl.utils.random(nacl.secret.SecretBox.KEY_SIZE)).decode()
FILE_UPLOAD_MAX_MEMORY_SIZE = config_get('FILE_UPLOAD_MAX_MEMORY_SIZE', global_settings.FILE_UPLOAD_MAX_MEMORY_SIZE)
DATA_UPLOAD_MAX_MEMORY_SIZE = config_get('DATA_UPLOAD_MAX_MEMORY_SIZE', global_settings.DATA_UPLOAD_MAX_MEMORY_SIZE)
FILE_UPLOAD_TEMP_DIR = config_get('FILE_UPLOAD_TEMP_DIR', global_settings.FILE_UPLOAD_TEMP_DIR)
AVAILABLE_FILESYSTEMS = {
'local': 'django.core.files.storage.FileSystemStorage',
'amazon_s3': 'storages.backends.s3boto3.S3Boto3Storage',
'azure': 'storages.backends.azure_storage.AzureStorage',
'dropbox': 'storages.backends.dropbox.DropBoxStorage',
'google_cloud': 'storages.backends.gcloud.GoogleCloudStorage',
}
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.sites',
'corsheaders',
'rest_framework',
'restapi',
'cron'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'corsheaders.middleware.CorsMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
PASSWORD_HASHERS = (
'django.contrib.auth.hashers.BCryptSHA256PasswordHasher',
'django.contrib.auth.hashers.BCryptPasswordHasher',
'django.contrib.auth.hashers.PBKDF2PasswordHasher',
'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher'
)
REST_FRAMEWORK = {
'DEFAULT_PERMISSION_CLASSES': (
'rest_framework.permissions.IsAdminUser',
),
'DEFAULT_PARSER_CLASSES': (
'restapi.parsers.DecryptJSONParser',
# 'rest_framework.parsers.FormParser', # default for Form Parsing
'rest_framework.parsers.MultiPartParser', # default for UnitTest Parsing
),
'DEFAULT_RENDERER_CLASSES': (
'restapi.renderers.EncryptJSONRenderer',
# 'rest_framework.renderers.BrowsableAPIRenderer',
),
'DEFAULT_THROTTLE_CLASSES': (
'rest_framework.throttling.AnonRateThrottle',
'rest_framework.throttling.UserRateThrottle',
'rest_framework.throttling.ScopedRateThrottle',
),
'DEFAULT_THROTTLE_RATES': {
'anon': '1440/day',
'user': '28800/day',
'health_check': '61/hour',
'cron': '61/minute',
'transfer': '61/minute'
},
}
LOGGING = {
# 'version': 1,
# 'disable_existing_loggers': False,
# 'formatters': {
# 'restapi_query_formatter': {
# '()': 'restapi.log.QueryFormatter',
# 'format': '%(time_utc)s logger=%(name)s, %(message)s'
# }
# },
# 'filters': {
# 'restapi_query_console': {
# '()': 'restapi.log.FilterQueryConsole',
# },
# },
# 'handlers': {
# 'restapi_query_handler_console': {
# 'level': 'DEBUG',
# 'class': 'logging.StreamHandler',
# 'formatter': 'restapi_query_formatter',
# 'filters': ['restapi_query_console'],
# },
# },
# 'loggers': {
# 'django.db.backends': {
# 'level': 'DEBUG',
# 'handlers': ['restapi_query_handler_console'],
# }
# }
}
for key, value in config_get('DEFAULT_THROTTLE_RATES', {}).items():
REST_FRAMEWORK['DEFAULT_THROTTLE_RATES'][key] = value # type: ignore
ROOT_URLCONF = 'psono.urls'
SITE_ID = 1
CORS_ORIGIN_ALLOW_ALL = True
CORS_ALLOW_CREDENTIALS = True
CORS_ALLOW_METHODS = (
'GET',
'POST',
'PUT',
'PATCH',
'DELETE',
'OPTIONS'
)
CORS_ALLOW_HEADERS = default_headers + (
'authorization-validator',
'pragma',
'if-modified-since',
'cache-control',
)
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'psono.wsgi.application'
# Database
# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ':memory:',
}
}
CACHE_ENABLE = config_get('CACHE_ENABLE', False)
if config_get('CACHE_DB', False):
CACHES = {
"default": {
"BACKEND": 'django.core.cache.backends.db.DatabaseCache',
"LOCATION": 'restapi_cache',
}
}
if config_get('CACHE_REDIS', False):
CACHES = {
"default": { # type: ignore
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": config_get('CACHE_REDIS_LOCATION', 'redis://localhost:6379/0'),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
}
}
if not config_get('THROTTLING', True):
CACHES = {
"default": {
"BACKEND": 'django.core.cache.backends.dummy.DummyCache',
}
}
TIME_SERVER = config_get('TIME_SERVER', 'time.google.com')
# Internationalization
# https://docs.djangoproject.com/en/2.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.0/howto/static-files/
STATIC_URL = '/static/'
with open(os.path.join(BASE_DIR, 'VERSION.txt')) as f:
VERSION = f.readline().rstrip()
SESSION_CRYPTO_BOX = nacl.secret.SecretBox(FILESERVER_SESSION_KEY, encoder=nacl.encoding.HexEncoder)
def generate_fileserver_info():
cluster_crypto_box = Box(PrivateKey(CLUSTER_PRIVATE_KEY, encoder=nacl.encoding.HexEncoder),
PublicKey(SERVER_PUBLIC_KEY, encoder=nacl.encoding.HexEncoder))
nonce = nacl.utils.random(nacl.secret.SecretBox.NONCE_SIZE)
encrypted = cluster_crypto_box.encrypt(json.dumps({
'CLUSTER_ID': CLUSTER_ID,
'FILESERVER_ID': FILESERVER_ID,
'FILESERVER_PUBLIC_KEY': PUBLIC_KEY,
'FILESERVER_SESSION_KEY': FILESERVER_SESSION_KEY,
'SHARDS_PUBLIC': SHARDS_PUBLIC,
'READ': READ,
'WRITE': WRITE,
'IP_READ_WHITELIST': IP_READ_WHITELIST,
'IP_WRITE_WHITELIST': IP_WRITE_WHITELIST,
'IP_READ_BLACKLIST': IP_READ_BLACKLIST,
'IP_WRITE_BLACKLIST': IP_WRITE_BLACKLIST,
'HOST_URL': HOST_URL,
}).encode("utf-8"), nonce)
return nacl.encoding.HexEncoder.encode(encrypted).decode()
FILESERVER_INFO = generate_fileserver_info()
def generate_signature():
info = {
'version': VERSION,
'fileserver_id': FILESERVER_ID,
'api': 1,
'public_key': PUBLIC_KEY,
'cluster_id': CLUSTER_ID,
'shards': SHARDS_PUBLIC,
'read': READ,
'write': WRITE,
'host_url': HOST_URL,
}
info = json.dumps(info)
signing_box = nacl.signing.SigningKey(PRIVATE_KEY, encoder=nacl.encoding.HexEncoder)
verify_key = signing_box.verify_key.encode(encoder=nacl.encoding.HexEncoder)
# The first 128 chars (512 bits or 64 bytes) are the actual signature, the rest the binary encoded info
signature = binascii.hexlify(signing_box.sign(six.b(info)))[:128]
return {
'info': info,
'signature': signature,
'verify_key': verify_key,
}
SIGNATURE = generate_signature()

31
psono/psono/urls.py Normal file
View File

@ -0,0 +1,31 @@
"""psono URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from rest_framework import routers
router = routers.DefaultRouter()
# Wire up our API using automatic URL routing.
# Additionally, we include login URLs for the browsable API.
urlpatterns = [
#url(r'^', include(router.urls)),
#url(r'^accounts/', include('allauth.urls')),
#url(r'^rest-auth/', include('rest_auth.urls')),
#url(r'^rest-auth/registration/', include('rest_auth.registration.urls')),
url(r'^', include('restapi.urls')),
url(r'^cron/', include('cron.urls')),
url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework'))
]

16
psono/psono/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for psono project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "psono.settings")
application = get_wsgi_application()

View File

3
psono/restapi/admin.py Normal file
View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

View File

@ -0,0 +1,22 @@
from django.conf import settings
from importlib import import_module
from .serializers import (
UploadSerializer as DefaultUploadSerializer,
)
def import_callable(path_or_callable):
if hasattr(path_or_callable, '__call__'):
return path_or_callable
else:
package, attr = path_or_callable.rsplit('.', 1)
return getattr(import_module(package), attr)
serializers = getattr(settings, 'RESTAPI_SERIALIZERS', {})
UploadSerializer = import_callable(
serializers.get('UPLOAD_SERIALIZER', DefaultUploadSerializer)
)

5
psono/restapi/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class RestapiConfig(AppConfig):
name = 'restapi'

24
psono/restapi/fields.py Normal file
View File

@ -0,0 +1,24 @@
from django.utils.translation import ugettext_lazy as _
from rest_framework.serializers import UUIDField as InsecureUUIDField
from rest_framework.serializers import BooleanField as InsecureBooleanField
from rest_framework.serializers import NullBooleanField as InsecureNullBooleanField
class UUIDField(InsecureUUIDField):
# Minimizes Reflected XSS
default_error_messages = {
'invalid': _('Is not a valid UUID.'),
}
class BooleanField(InsecureBooleanField):
# Minimizes Reflected XSS
default_error_messages = {
'invalid': _('Is not a valid boolean.')
}
class NullBooleanField(InsecureNullBooleanField):
# Minimizes Reflected XSS
default_error_messages = {
'invalid': _('Is not a valid boolean.')
}

View File

3
psono/restapi/models.py Normal file
View File

@ -0,0 +1,3 @@
from django.db import models
# Create your models here.

133
psono/restapi/parsers.py Normal file
View File

@ -0,0 +1,133 @@
"""
Parsers are used to parse the content of incoming HTTP requests.
They give us a generic way of being able to handle various media types
on the request, such as form content or json encoded data.
"""
from __future__ import unicode_literals
from django.conf import settings
from django.core.files.uploadhandler import StopFutureHandlers
from django.http.multipartparser import (
ChunkIter, parse_header
)
from rest_framework.parsers import JSONParser, BaseParser, DataAndFiles
from rest_framework import renderers
from rest_framework.exceptions import ParseError
import nacl.encoding
import nacl.secret
import json
def decrypt(session_secret_key, text_hex, nonce_hex):
text = nacl.encoding.HexEncoder.decode(text_hex)
nonce = nacl.encoding.HexEncoder.decode(nonce_hex)
secret_box = nacl.secret.SecretBox(session_secret_key, encoder=nacl.encoding.HexEncoder)
return secret_box.decrypt(text, nonce)
class DecryptJSONParser(JSONParser):
"""
Decrypts data after JSON deserialization.
"""
media_type = 'application/json'
renderer_class = renderers.JSONRenderer
def parse(self, stream, media_type=None, parser_context=None):
"""
Takes the incoming JSON object, and decrypts the data
"""
data = super(DecryptJSONParser, self).parse(stream, media_type, parser_context)
if 'text' not in data or 'nonce' not in data:
return data
decrypted_data = decrypt(stream.auth.secret_key, data['text'], data['nonce'])
try:
data = json.loads(decrypted_data.decode())
except ValueError:
raise ParseError('Invalid request')
return data
class FileUploadParser(BaseParser):
"""
Parser for file upload data.
"""
media_type = '*/*'
errors = {
'unhandled': 'FileUpload parse error - none of upload handlers can handle the stream',
}
def parse(self, stream, media_type=None, parser_context=None):
"""
Treats the incoming bytestream as a raw file upload and returns
a `DataAndFiles` object.
`.data` will be None (we expect request body to be a file content).
`.files` will be a `QueryDict` containing one 'file' element.
"""
parser_context = parser_context or {}
request = parser_context['request']
encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)
meta = request.META
upload_handlers = request.upload_handlers
# Note that this code is extracted from Django's handling of
# file uploads in MultiPartParser.
content_type = meta.get('HTTP_CONTENT_TYPE',
meta.get('CONTENT_TYPE', ''))
try:
content_length = int(meta.get('HTTP_CONTENT_LENGTH',
meta.get('CONTENT_LENGTH', 0)))
except (ValueError, TypeError):
content_length = None
# See if the handler will want to take care of the parsing.
for handler in upload_handlers:
result = handler.handle_raw_input(stream,
meta,
content_length,
None,
encoding)
if result is not None:
return DataAndFiles({}, {'file': result[1]})
# This is the standard case.
possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size]
chunk_size = min([2 ** 31 - 4] + possible_sizes)
chunks = ChunkIter(stream, chunk_size)
counters = [0] * len(upload_handlers)
for index, handler in enumerate(upload_handlers):
try:
handler.new_file(None, 'dummy', content_type,
content_length, encoding)
except StopFutureHandlers:
upload_handlers = upload_handlers[:index + 1]
break
for chunk in chunks:
for index, handler in enumerate(upload_handlers):
chunk_length = len(chunk)
chunk = handler.receive_data_chunk(chunk, counters[index])
counters[index] += chunk_length
if chunk is None:
break
for index, handler in enumerate(upload_handlers):
file_obj = handler.file_complete(counters[index])
if file_obj is not None:
return DataAndFiles({}, {
'file': file_obj,
})
raise ParseError(self.errors['unhandled'])

View File

@ -0,0 +1,67 @@
from __future__ import unicode_literals
import six
import nacl.encoding
import nacl.utils
import nacl.secret
from rest_framework.renderers import JSONRenderer
from rest_framework.settings import api_settings
from rest_framework.utils import encoders
def encrypt(session_secret_key, msg):
# generate random nonce
nonce = nacl.utils.random(nacl.secret.SecretBox.NONCE_SIZE)
# open crypto box with session secret
secret_box = nacl.secret.SecretBox(session_secret_key, encoder=nacl.encoding.HexEncoder)
# encrypt msg with crypto box and nonce
encrypted = secret_box.encrypt(msg, nonce)
# cut away the nonce
text = encrypted[len(nonce):]
# convert nonce and encrypted msg to hex
nonce_hex = nacl.encoding.HexEncoder.encode(nonce)
text_hex = nacl.encoding.HexEncoder.encode(text)
return {'text': text_hex, 'nonce': nonce_hex}
class EncryptJSONRenderer(JSONRenderer):
"""
Renderer which encrypts JSON serialized objects.
"""
media_type = 'application/json'
format = 'json'
encoder_class = encoders.JSONEncoder
ensure_ascii = not api_settings.UNICODE_JSON
compact = api_settings.COMPACT_JSON
# We don't set a charset because JSON is a binary encoding,
# that can be encoded as utf-8, utf-16 or utf-32.
# See: http://www.ietf.org/rfc/rfc4627.txt
# Also: http://lucumr.pocoo.org/2013/7/19/application-mimetypes-and-encodings/
charset = None
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Render `data` into JSON, returning a bytestring.
"""
decrypted_data = super(EncryptJSONRenderer, self).render(data, accepted_media_type, renderer_context)
if renderer_context['request'].auth is None:
return decrypted_data
if decrypted_data == six.b(''):
return decrypted_data
session_secret_key = renderer_context['request'].auth.secret_key
encrypted_data = encrypt(session_secret_key, decrypted_data)
decrypted_data_json = super(EncryptJSONRenderer, self).render(encrypted_data, accepted_media_type, renderer_context)
return decrypted_data_json

View File

@ -0,0 +1 @@
from .upload import UploadSerializer

View File

@ -0,0 +1,65 @@
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers, exceptions
from restapi.fields import UUIDField
from urllib.parse import urlencode
class UploadSerializer(serializers.Serializer):
# duo_id = UUIDField(required=True)
# duo_token = serializers.CharField(max_length=6, min_length=6, required=False)
def validate(self, attrs: dict) -> dict:
# duo_id = attrs.get('duo_id')
# duo_token = attrs.get('duo_token', None)
#
# try:
# duo = Duo.objects.get(pk=duo_id, user=self.context['request'].user, active=False)
# except Duo.DoesNotExist:
# msg = _("You don't have permission to access or it does not exist.")
# raise exceptions.ValidationError(msg)
#
# enrollment_status = duo_auth_enroll_status(duo.duo_integration_key, decrypt_with_db_secret(duo.duo_secret_key), duo.duo_host, duo.enrollment_user_id, duo.enrollment_activation_code)
#
# if enrollment_status == 'invalid':
# duo.delete()
# msg = _("Duo enrollment expired")
# raise exceptions.ValidationError(msg)
#
# if enrollment_status == 'waiting':
# # Pending activation
# msg = _("Scan the barcode first")
# raise exceptions.ValidationError(msg)
#
# if duo_token is not None:
# factor = 'passcode'
# device = None
# else:
# factor = 'push'
# device = 'auto'
#
# username, domain = self.context['request'].user.username.split("@")
#
# duo_auth_return = duo_auth_auth(
# integration_key=duo.duo_integration_key,
# secret_key=decrypt_with_db_secret(duo.duo_secret_key),
# host=duo.duo_host,
# user_id=duo.enrollment_user_id,
# factor=factor,
# device=device,
# pushinfo=urlencode({'Host': domain}),
# passcode=duo_token
# )
#
# if 'result' not in duo_auth_return or duo_auth_return['result'] != 'allow':
# if 'status_msg' in duo_auth_return:
# msg = _(duo_auth_return['status_msg'])
# elif 'error' in duo_auth_return:
# msg = _(duo_auth_return['error'])
# else:
# msg = _('Validation failed.')
# raise exceptions.ValidationError(msg)
#
# attrs['duo'] = duo
return attrs

View File

@ -0,0 +1,3 @@
from .health_check import *
from .info import *

View File

@ -0,0 +1,38 @@
from rest_framework.test import APITestCase
from uuid import UUID
def is_uuid(expr):
"""
check if a given expression is a uuid (version 4)
:param expr: the possible uuid
:return: True or False
:rtype: bool
"""
try:
val = UUID(expr, version=4)
except ValueError:
val = False
return not not val
class APITestCaseExtended(APITestCase):
@staticmethod
def safe_repr(self, obj, short=False):
_MAX_LENGTH = 80
try:
result = repr(obj)
except Exception:
result = object.__repr__(obj)
if not short or len(result) < _MAX_LENGTH:
return result
return result[:_MAX_LENGTH] + ' [truncated]...'
def assertIsUUIDString(self, expr, msg=None):
"""Check that the expression is a valid uuid"""
if not is_uuid(expr):
msg = self._formatMessage(msg, "%s is not an uuid" % self.safe_repr(expr))
raise self.failureException(msg)

View File

@ -0,0 +1,66 @@
from django.urls import reverse
from rest_framework import status
from .base import APITestCaseExtended
from mock import patch
from restapi import models
class HealthCheckTest(APITestCaseExtended):
"""
Test for health check
"""
def test_put_healthcheckn(self):
"""
Tests PUT method on healthcheck
"""
url = reverse('healthcheck')
data = {}
response = self.client.put(url, data)
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
def test_post_healthcheckn(self):
"""
Tests POST method on healthcheck
"""
url = reverse('healthcheck')
data = {}
response = self.client.post(url, data)
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
def test_delete_healthcheckn(self):
"""
Tests DELETE method on healthcheck
"""
url = reverse('healthcheck')
data = {}
response = self.client.delete(url, data)
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
def test_get_healthcheckn(self):
"""
Tests GET method on healthcheck
"""
url = reverse('healthcheck')
data = {}
response = self.client.get(url, data)
self.assertEqual(response.status_code, status.HTTP_200_OK)

View File

@ -0,0 +1,78 @@
from django.urls import reverse
from django.conf import settings
from rest_framework import status
from .base import APITestCaseExtended
import json
class ReadInfoTest(APITestCaseExtended):
"""
Test to read info ressource
"""
def test_read_info_success(self):
"""
Tests to read all groups
"""
url = reverse('info')
data = {}
response = self.client.get(url, data)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertNotEqual(response.data.get('verify_key', None), None)
self.assertNotEqual(response.data.get('info', None), None)
self.assertNotEqual(response.data.get('signature', None), None)
info = json.loads(response.data.get('info'))
self.assertNotEqual(info.get('version', None), None)
self.assertNotEqual(info.get('public_key', None), None)
self.assertNotEqual(info.get('api', None), None)
self.assertEqual(info.get('version', None), settings.VERSION)
self.assertEqual(info.get('public_key', None), settings.PUBLIC_KEY)
def test_put_info(self):
"""
Tests PUT request on info
"""
url = reverse('info')
data = {}
response = self.client.put(url, data)
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
def test_post_info(self):
"""
Tests POST request on info
"""
url = reverse('info')
data = {}
response = self.client.post(url, data)
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
def test_delete_info(self):
"""
Tests DELETE request on info
"""
url = reverse('info')
data = {}
response = self.client.delete(url, data)
self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)

37
psono/restapi/urls.py Normal file
View File

@ -0,0 +1,37 @@
"""psono URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Add an import: from blog import urls as blog_urls
2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls))
"""
from django.conf.urls import url
from django.conf import settings
from os.path import join, dirname, abspath
import django
from . import views
urlpatterns = [
# url(r'^$', views.api_root),
url(r'^healthcheck/$', views.HealthCheckView.as_view(), name='healthcheck'),
url(r'^upload/$', views.UploadView.as_view(), name='upload'),
url(r'^info/$', views.InfoView.as_view(), name='info'),
]
if settings.DEBUG:
# URLs for development purposes only
urlpatterns += [
url(r'^coverage/(?P<path>.*)$', django.views.static.serve,
{'document_root':join(dirname(abspath(__file__)), '..', '..', 'htmlcov')}),
]

View File

@ -0,0 +1,3 @@
from .api_server import *

View File

@ -0,0 +1,48 @@
from django.conf import settings
import requests
import json
import nacl.encoding
import nacl.secret
class APIServer(object):
@staticmethod
def _decrypt(r):
r.json_decrypted = None
try:
json_encrypted = json.loads(r.text)
text = nacl.encoding.HexEncoder.decode(json_encrypted['text'])
nonce = nacl.encoding.HexEncoder.decode(json_encrypted['nonce'])
decrypted_text = settings.SESSION_CRYPTO_BOX.decrypt(text, nonce)
r.json_decrypted = json.loads(decrypted_text.decode())
except:
pass
@staticmethod
def query(endpoint, data=None, headers=None):
if not data:
data = {}
if not headers:
headers = {
'Authorization': 'Token ' + settings.FILESERVER_ID,
'Authorization-Validator': json.dumps({
'fileserver_info': settings.FILESERVER_INFO,
'cluster_id': settings.CLUSTER_ID
})
}
r = requests.put(settings.SERVER_URL + endpoint, data=data, verify=settings.SERVER_URL_VERIFY_SSL, headers=headers)
APIServer._decrypt(r)
return r

View File

@ -0,0 +1,3 @@
from .health_check import HealthCheckView
from .upload import UploadView
from .info import InfoView

View File

@ -0,0 +1,65 @@
from rest_framework import status
from rest_framework.response import Response
from rest_framework.generics import GenericAPIView
from rest_framework.permissions import AllowAny
from django.conf import settings
import ntplib
class HealthCheckView(GenericAPIView):
permission_classes = (AllowAny,)
allowed_methods = ('GET', 'OPTIONS', 'HEAD')
throttle_scope = 'health_check'
def get(self, request, *args, **kwargs):
"""
Check the health of the application
:param request:
:type request:
:param args:
:type args:
:param kwargs:
:type kwargs:
:return:
:rtype:
"""
unhealthy = False
time_sync = True
not_debug_mode = True
def time_sync_unhealthy():
c = ntplib.NTPClient()
response = c.request(settings.TIME_SERVER, version=3)
return abs(response.offset) > 1
if time_sync_unhealthy():
unhealthy = True
time_sync = False
if not settings.DEBUG:
# unhealthy = True
not_debug_mode = False
if unhealthy:
health_status = status.HTTP_400_BAD_REQUEST
else:
health_status = status.HTTP_200_OK
return Response({
'time_sync': {'healthy': time_sync},
'debug_mode': {'healthy': not_debug_mode},
}, status=health_status)
def put(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def post(self, request, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def delete(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)

View File

@ -0,0 +1,35 @@
from rest_framework import status
from rest_framework.response import Response
from rest_framework.generics import GenericAPIView
from rest_framework.permissions import AllowAny
from django.conf import settings
class InfoView(GenericAPIView):
permission_classes = (AllowAny,)
allowed_methods = ('GET', 'OPTIONS', 'HEAD')
throttle_scope = 'health_check'
def get(self, request, *args, **kwargs):
"""
Returns the Server's signed information
:param request:
:type request:
:param args:
:type args:
:param kwargs:
:type kwargs:
:return:
:rtype:
"""
return Response(settings.SIGNATURE, status=status.HTTP_200_OK)
def put(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def post(self, request, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def delete(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)

View File

@ -0,0 +1,63 @@
from django.conf import settings
from django.core.files.base import ContentFile
from django.core.files.storage import get_storage_class
from rest_framework import status
from rest_framework.response import Response
from rest_framework.generics import GenericAPIView
from rest_framework.permissions import AllowAny
from ..parsers import FileUploadParser
from ..app_settings import UploadSerializer
import os
import pyblake2
class UploadView(GenericAPIView):
parser_classes = (FileUploadParser,)
permission_classes = (AllowAny,)
allowed_methods = ('POST', 'OPTIONS', 'HEAD')
throttle_scope = 'transfer'
def get(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def put(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
def post(self, request, *args, **kwargs):
serializer = UploadSerializer(data=request.data, context=self.get_serializer_context())
if not serializer.is_valid():
return Response(
serializer.errors, status=status.HTTP_400_BAD_REQUEST
)
# shard_id = serializer.validated_data['shard_id']
shard_id = '338b17b6-07d1-432c-ab46-732044074f68'
shard_config = settings.SHARDS_DICT[shard_id]
# TODO IMPORTANT. Enforce hex in hash_blake2b
# TODO Test user quota
# TODO Test if shard exist
# TODO Test if write is allwoed for this shard
# TODO Test if write is allwoed for this IP
storage = get_storage_class(settings.AVAILABLE_FILESYSTEMS[shard_config['engine']['class']])(**shard_config['engine']['kwargs'])
file_content = request.data['file'].read()
request.data['file'].seek(0)
hash_blake2b = pyblake2.blake2b(file_content).hexdigest()
target_path = os.path.join(hash_blake2b[0:2], hash_blake2b[2:4], hash_blake2b)
if not storage.exists(target_path):
storage.save(target_path, ContentFile(request.data['file'].read()))
return Response({}, status=status.HTTP_200_OK)
def delete(self, *args, **kwargs):
return Response({}, status=status.HTTP_405_METHOD_NOT_ALLOWED)

2
requirements-dev.txt Normal file
View File

@ -0,0 +1,2 @@
coverage
mock

20
requirements.txt Normal file
View File

@ -0,0 +1,20 @@
six
cffi
django
djangorestframework
django-rest-auth
django-allauth
django-cors-headers
markdown
django-filter
pyyaml
more_itertools
pynacl
django-redis
ntplib
python-dateutil
django-storages
dropbox
azure
apache-libcloud
pyblake2

6
var/backup/.env Normal file
View File

@ -0,0 +1,6 @@
PSONO_BACKUP_PATH_TO_SETTINGS_YML=/path/to/settings.yaml
PSONO_BACKUP_DATABASE_NAME=replace_me
PSONO_BACKUP_DATABASE_USER=replace_me
PSONO_BACKUP_DATABASE_PASSWORD=replace_me
PSONO_BACKUP_TIMESTAMP=$(date +%F_%R)
PSONO_BACKUP_PATH=/path/to/backup/folder

12
var/backup/backup Normal file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
source .env
export PGPASSWORD="$PSONO_BACKUP_DATABASE_PASSWORD"
backup_folder_running="${PSONO_BACKUP_PATH}/backup_${PSONO_BACKUP_TIMESTAMP}.running"
backup_folder_complete="${PSONO_BACKUP_PATH}/backup_${PSONO_BACKUP_TIMESTAMP}.complete"
mkdir -p $backup_folder_running
pg_dump -U $PSONO_BACKUP_DATABASE_USER $PSONO_BACKUP_DATABASENAME | gzip > "${backup_folder_running}/db.sql.gz"
cp $PSONO_BACKUP_PATH_TO_SETTINGS_YML "${backup_folder_running}/settings.yaml"
mv "$backup_folder_running" "$backup_folder_complete"

70
var/backup/restore Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env bash
source .env
# http://stackoverflow.com/a/39398359/4582775
# As long as there is at least one more argument, keep looping
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# This is an arg value type option. Will catch -o value or --output-file value
-b|--backup)
shift # past the key and to the value
backup_folder="$1"
;;
# This is an arg=value type option. Will catch -o=value or --output-file=value
-b=*|--backup=*)
# No need to shift here since the value is part of the same string
backup_folder="${key#*=}"
;;
*)
# Do whatever you want with extra options
echo "Unknown option '$key'"
;;
esac
# Shift after checking all the cases to get the next option
shift
done
if [ -z "$backup_folder" ]; then
echo -e "backup variable not specified. usage: \n ./restore --backup=/path/to/backup_12345..." >&2
exit 1
fi
# Ensure trailing slash
backup_folder=${backup_folder%/}/
db_file="${backup_folder}db.sql.gz"
settings_file="${backup_folder}settings.yaml"
errors=0
if [ ! -f "$db_file" ]; then
errors=1
echo "No valid backup, db.sql.gz is missing."
fi
if [ ! -f "$settings_file" ]; then
errors=1
echo "No valid backup, settings.yaml is missing."
fi
if ! psql -U "$PSONO_BACKUP_DATABASE_USER" -lqt | cut -d \| -f 1 | grep -qw "$PSONO_BACKUP_DATABASE_NAME"; then
errors=1
echo "Database does not exist."
fi
if [ "$( psql -U "$PSONO_BACKUP_DATABASE_USER" $PSONO_BACKUP_DATABASE_NAME -tAc "SELECT 1 FROM information_schema.tables WHERE table_name = 'django_migrations'" )" = '1' ]
then
errors=1
echo "Database already has another django installation. Please delete all content first."
fi
if [ "$errors" -eq "1" ]; then
echo -e "Errors detected, aborted." >&2
exit 1
fi
gunzip -c $db_file | psql -U "$PSONO_BACKUP_DATABASE_USER" $PSONO_BACKUP_DATABASE_NAME
cp "$settings_file" "$PSONO_BACKUP_PATH_TO_SETTINGS_YML"
echo "Backup restored."

39
var/deploy.sh Normal file
View File

@ -0,0 +1,39 @@
#!/usr/bin/env bash
apk upgrade --no-cache
apk add --update curl
# Deploy to Docker Hub
docker pull psono-docker.jfrog.io/psono/psono-fileserver:latest
docker tag psono-docker.jfrog.io/psono/psono-fileserver:latest psono/psono-fileserver:latest
docker push psono/psono-fileserver:latest
# Inform production stage about new image
curl -X POST https://hooks.microbadger.com/images/psono/psono-fileserver/8BDLpDMSMHR-Ias4JAPRhy0f-cg=
curl -X POST $psono_image_updater_url
# Deploy to GitHub
echo "Clonging gitlab.com/psono/psono-fileserver.git"
git clone https://gitlab.com/psono/psono-fileserver.git
cd psono-fileserver
git branch --track develop origin/develop
git fetch --all
git pull --all
echo "Empty .ssh folder"
if [ -d "/root/.ssh" ]; then
rm -Rf /root/.ssh;
fi
mkdir -p /root/.ssh
echo "Fill .ssh folder"
echo "$github_deploy_key" > /root/.ssh/id_rsa
cat > /root/.ssh/known_hosts <<- "EOF"
|1|QihaxuxIU4rUFjd+Zi5Mr3V0oyI=|m1minLYaqd2pSUN52YJk1ROukfY= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
|1|dhFyBNrE6k3jSyFFOoEbeJKgbcs=|W0ag0VmyD+G4NSRpMOGkApaY594= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
EOF
chmod 600 /root/.ssh/id_rsa
chmod 600 /root/.ssh/known_hosts
echo "Push to github.com/psono/psono-fileserver.git"
git remote set-url origin git@github.com:psono/psono-fileserver.git
git push --all origin

11
var/deploy_changelog.sh Normal file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env bash
apt-get update && \
apt-get install -y lsb-release curl && \
export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
apt-get update && apt-get -y install google-cloud-sdk && \
echo "$GOOGLE_APPLICATION_CREDENTIALS" > "/root/key.json" && \
gcloud auth activate-service-account --key-file=/root/key.json && \
curl -H "PRIVATE-TOKEN: $GITLAB_PERSONAL_ACCESS_TOKEN" "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/tags" --output changelog.json && \
gsutil cp changelog.json gs://static.psono.com/gitlab.com/$CI_PROJECT_PATH/changelog.json

17
var/update_version.sh Normal file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
if [ -z "$CI_COMMIT_TAG" ]; then
exit 0
fi
if [ -z "$CI_COMMIT_SHA" ]; then
exit 0
fi
if ! echo "$CI_COMMIT_TAG" | egrep -q ^v[0-9]+\.[0-9]+\.[0-9]+$; then
exit 0
fi
version="$(echo $CI_COMMIT_TAG | awk '{ string=substr($0, 2, 100); print string; }' ) (Build $(echo $CI_COMMIT_SHA | awk '{ string=substr($0, 1, 8); print string; }' ))"
echo $version > ./psono/VERSION.txt