diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index beb864f..57d8f72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ stages: - test - lint - release - - deploy variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -17,9 +16,8 @@ variables: cache: key: "$CI_COMMIT_REF_SLUG" paths: - - .venv/ + - env/ - .cache/pip - - .cache/poetry - node_modules/ include: @@ -27,4 +25,3 @@ include: - local: '/gitlab-ci/test.yml' - local: '/gitlab-ci/lint.yml' - local: '/gitlab-ci/release.yml' - - local: '/gitlab-ci/deploy.yml' diff --git a/bin/docker-entrypoint.sh b/bin/docker-entrypoint.sh new file mode 100755 index 0000000..356485f --- /dev/null +++ b/bin/docker-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +python /app/src/manage.py migrate + +exec "$@" diff --git a/config/nginx/conf.d/local.conf b/config/nginx/conf.d/local.conf new file mode 100644 index 0000000..45bba56 --- /dev/null +++ b/config/nginx/conf.d/local.conf @@ -0,0 +1,18 @@ +upstream gunicorn { + server django:8000; +} + +server { + listen 80; + server_name localhost; + + access_log /var/log/nginx/access_log; + error_log /var/log/nginx/error_log; + + location / { + proxy_pass http://gunicorn; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } +} diff --git a/docker-compose.development.yml b/docker-compose.development.yml new file mode 100644 index 0000000..5dd4932 --- /dev/null +++ b/docker-compose.development.yml @@ -0,0 +1,36 @@ +version: '3.6' + +volumes: + static-files: + node-modules: + +services: + celery: + environment: + - DJANGO_SETTINGS_MODULE=newsreader.conf.docker + volumes: + - ./src/:/app/src + + django: + build: + context: . + dockerfile: ./docker/django + target: development + command: python /app/src/manage.py runserver 0.0.0.0:8000 + environment: + - DJANGO_SETTINGS_MODULE=newsreader.conf.docker + volumes: + - ./src:/app/src + - static-files:/app/src/newsreader/static + stdin_open: true + tty: true + + webpack: + build: + context: . + dockerfile: ./docker/webpack + command: npm run build:watch + volumes: + - ./src/:/app/src + - static-files:/app/src/newsreader/static + - node-modules:/app/node_modules diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 0000000..ed92cde --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,14 @@ +version: '3.6' + +volumes: + logs: + +services: + nginx: + image: nginx:1.23 + depends_on: + django: + condition: service_healthy + volumes: + - ./config/nginx/conf.d:/etc/nginx/conf.d + - logs:/var/log/nginx diff --git a/docker-compose.yml b/docker-compose.yml index 07b1c2c..4bf7f5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,62 +1,84 @@ -version: "3" +version: '3.6' + volumes: + logs: + media: postgres-data: - static-files: - node-modules: services: db: - image: postgres + image: postgres:15 + healthcheck: + test: /usr/bin/pg_isready + interval: 5s + timeout: 10s + retries: 10 environment: - POSTGRES_DB: "newsreader" - POSTGRES_USER: "newsreader" - POSTGRES_PASSWORD: "newsreader" + POSTGRES_DB: 'newsreader' + POSTGRES_USER: 'newsreader' + POSTGRES_PASSWORD: 'newsreader' volumes: - postgres-data:/var/lib/postgresql/data + rabbitmq: image: rabbitmq:3.7 + memcached: image: memcached:1.6 ports: - - "11211:11211" + - '11211:11211' entrypoint: - memcached - -m 64 + celery: build: context: . dockerfile: ./docker/django - command: celery worker -n worker1@%h -n worker2@%h --app newsreader --loglevel INFO --concurrency 2 --workdir /app/src/ --beat --scheduler django + command: | + celery worker -n worker1@%h + -n worker2@%h + --app newsreader + --loglevel INFO + --concurrency 2 + --workdir /app/src/ + --beat + --scheduler django environment: - - DJANGO_SETTINGS_MODULE=newsreader.conf.docker + - DJANGO_SETTINGS_MODULE=newsreader.conf.production depends_on: - - rabbitmq - - memcached + rabbitmq: + condition: service_started + memcached: + condition: service_started + db: + condition: service_healthy + django: + condition: service_healthy volumes: - - ./src/:/app/src + - logs:/app/logs + django: build: context: . dockerfile: ./docker/django - command: python /app/src/manage.py runserver 0.0.0.0:8000 + target: production + entrypoint: /app/bin/docker-entrypoint.sh + command: gunicorn --bind 0.0.0.0:8000 --workers 3 newsreader.wsgi:application + healthcheck: + test: /usr/bin/curl --fail http://django:8000 || exit 1 + interval: 5s + timeout: 10s + retries: 10 environment: - - DJANGO_SETTINGS_MODULE=newsreader.conf.docker + - DJANGO_SETTINGS_MODULE=newsreader.conf.production ports: - - "8000:8000" + - '8000:8000' depends_on: - - db - - memcached + memcached: + condition: service_started + db: + condition: service_healthy volumes: - - ./src:/app/src - - static-files:/app/src/newsreader/static - stdin_open: true - tty: true - webpack: - build: - context: . - dockerfile: ./docker/webpack - command: npm run build:watch - volumes: - - ./src/:/app/src - - static-files:/app/src/newsreader/static - - node-modules:/app/node_modules + - logs:/app/logs + - media:/app/media diff --git a/docker/django b/docker/django index b9c47da..c81bd5d 100644 --- a/docker/django +++ b/docker/django @@ -1,10 +1,105 @@ -FROM python:3.9-bullseye +# stage 1 +FROM python:3.9-bullseye as backend + +RUN apt-get update && apt-get install -y --no-install-recommends \ + vim \ + curl \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app RUN mkdir /app/src +RUN mkdir /app/logs +RUN mkdir /app/media COPY ./requirements /app/requirements -RUN pip install -r requirements/production.txt -r requirements/development.txt +RUN pip install -r requirements/production.txt + + +# stage 2 +FROM node:current-bullseye AS frontend-build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY ./build /app/build/ +COPY ./*.json ./*.js ./.babelrc /app/ + +RUN npm ci COPY ./src /app/src + +RUN npm run build + + +# stage 3 +FROM python:3.9-bullseye as production + +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-client \ + vim \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +RUN mkdir /app/logs +RUN mkdir /app/media +RUN mkdir /app/bin + +COPY --from=backend /usr/local/lib/python3.9 /usr/local/lib/python3.9 +COPY --from=backend /usr/local/bin/gunicorn /usr/local/bin/gunicorn +COPY --from=backend /usr/local/bin/celery /usr/local/bin/celery +COPY --from=backend /app/src/ /app/src/ +COPY ./bin/docker-entrypoint.sh /app/bin/docker-entrypoint.sh + +COPY --from=frontend-build /app/src/newsreader/static /app/src/newsreader/static + +COPY ./src /app/src + +RUN useradd -M -u 1000 newsreader +RUN chown -R newsreader:newsreader /app + +USER newsreader + +ARG COMMIT_HASH +ARG RELEASE=latest + +ENV RELEASE=${RELEASE} \ + GIT_SHA=${COMMIT_HASH} \ + PYTHONUNBUFFERED=1 \ + DJANGO_SETTINGS_MODULE=newsreader.conf.production + +ARG SECRET_KEY=dummy + +RUN python src/manage.py collectstatic --noinput \ + && python src/manage.py compilemessages + + +# (optional) stage 4 +FROM python:3.9-bullseye as development + +RUN apt-get update && apt-get install -y --no-install-recommends \ + vim \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +RUN mkdir /app/bin + +COPY ./requirements /app/requirements +COPY ./bin/docker-entrypoint.sh /app/bin/docker-entrypoint.sh +COPY --from=backend /usr/local/lib/python3.9 /usr/local/lib/python3.9 +COPY --from=backend /usr/local/bin/celery /usr/local/bin/celery +COPY --from=backend /app/src/ /app/src/ + +RUN pip install -r requirements/production.txt -r requirements/development.txt + +RUN useradd -M -u 1000 newsreader +RUN chown -R newsreader:newsreader /app + +USER newsreader diff --git a/gitlab-ci/deploy.yml b/gitlab-ci/deploy.yml deleted file mode 100644 index ba0ba46..0000000 --- a/gitlab-ci/deploy.yml +++ /dev/null @@ -1,22 +0,0 @@ -deploy: - stage: deploy - image: python:3.7 - environment: - name: production - url: rss.fudiggity.nl - rules: - - if: $CI_COMMIT_TAG - before_script: - - pip install ansible --quiet - - git clone https://git.fudiggity.nl/ansible/newsreader.git deployment --branch master - - cd deployment - - ansible-galaxy install -r requirements.yml - - mkdir /root/.ssh && echo "$DEPLOY_HOST_KEY" > /root/.ssh/known_hosts - - echo "$DEPLOY_KEY" > deploy_key && chmod 0600 deploy_key - - echo "$VAULT_PASSWORD" > vault - script: - - > - ansible-playbook playbook.yml - --private-key deploy_key - --vault-password-file vault - --extra-vars "app_branch=$CI_COMMIT_TAG" diff --git a/gitlab-ci/test.yml b/gitlab-ci/test.yml index b100ff0..9547127 100644 --- a/gitlab-ci/test.yml +++ b/gitlab-ci/test.yml @@ -2,7 +2,7 @@ python-tests: stage: test coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' services: - - postgres:11 + - postgres:15 - memcached:1.5.22 image: python:3.9-bullseye before_script: diff --git a/src/newsreader/conf/base.py b/src/newsreader/conf/base.py index cd51218..f609cb5 100644 --- a/src/newsreader/conf/base.py +++ b/src/newsreader/conf/base.py @@ -21,7 +21,7 @@ DJANGO_PROJECT_DIR = os.path.join(BASE_DIR, "src", "newsreader") # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ # SECURITY WARNING: don"t run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ["127.0.0.1", "localhost"] INTERNAL_IPS = ["127.0.0.1", "localhost"] @@ -208,9 +208,6 @@ STATICFILES_FINDERS = [ # Email EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" -# Project settings -ENVIRONMENT = "development" - # Reddit integration REDDIT_CLIENT_ID = "CLIENT_ID" REDDIT_CLIENT_SECRET = "CLIENT_SECRET" @@ -261,7 +258,6 @@ ACCOUNT_ACTIVATION_DAYS = 7 SENTRY_CONFIG = { "dsn": os.environ.get("SENTRY_DSN"), "send_default_pii": False, - "environment": ENVIRONMENT, "integrations": [DjangoIntegration(), CeleryIntegration()] if DjangoIntegration and CeleryIntegration else [], diff --git a/src/newsreader/conf/docker.py b/src/newsreader/conf/docker.py index c141636..515ee64 100644 --- a/src/newsreader/conf/docker.py +++ b/src/newsreader/conf/docker.py @@ -2,6 +2,8 @@ from .base import * # isort:skip from .version import get_current_version +ALLOWED_HOSTS = ["django", "127.0.0.1"] + SECRET_KEY = "=q(ztyo)b6noom#a164g&s9vcj1aawa^g#ing_ir99=_zl4g&$" INSTALLED_APPS += ["debug_toolbar", "django_extensions"] @@ -31,6 +33,8 @@ CACHES = { }, } +DEBUG = True + # Project settings VERSION = get_current_version() ENVIRONMENT = "docker"