Initial refactor

.gitlab-ci.yml

@@ -3,7 +3,6 @@ stages:
   - test
   - lint
   - release
-  - deploy
 
 variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -17,9 +16,8 @@ variables:
 cache:
   key: "$CI_COMMIT_REF_SLUG"
   paths:
-    - .venv/
+    - env/
     - .cache/pip
-    - .cache/poetry
     - node_modules/
 
 include:
@@ -27,4 +25,3 @@ include:
   - local: '/gitlab-ci/test.yml'
   - local: '/gitlab-ci/lint.yml'
   - local: '/gitlab-ci/release.yml'
-  - local: '/gitlab-ci/deploy.yml'

bin/docker-entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+python /app/src/manage.py migrate
+
+exec "$@"

config/nginx/conf.d/local.conf

@@ -0,0 +1,18 @@
+upstream gunicorn {
+    server django:8000;
+}
+
+server {
+    listen 80;
+    server_name localhost;
+
+    access_log /var/log/nginx/access_log;
+    error_log /var/log/nginx/error_log;
+
+    location / {
+        proxy_pass http://gunicorn;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+}

docker-compose.development.yml

@@ -0,0 +1,36 @@
+version: '3.6'
+
+volumes:
+  static-files:
+  node-modules:
+
+services:
+  celery:
+    environment:
+      - DJANGO_SETTINGS_MODULE=newsreader.conf.docker
+    volumes:
+      - ./src/:/app/src
+
+  django:
+    build:
+      context: .
+      dockerfile: ./docker/django
+      target: development
+    command: python /app/src/manage.py runserver 0.0.0.0:8000
+    environment:
+      - DJANGO_SETTINGS_MODULE=newsreader.conf.docker
+    volumes:
+      - ./src:/app/src
+      - static-files:/app/src/newsreader/static
+    stdin_open: true
+    tty: true
+
+  webpack:
+    build:
+      context: .
+      dockerfile: ./docker/webpack
+    command: npm run build:watch
+    volumes:
+      - ./src/:/app/src
+      - static-files:/app/src/newsreader/static
+      - node-modules:/app/node_modules

docker-compose.production.yml

@@ -0,0 +1,14 @@
+version: '3.6'
+
+volumes:
+  logs:
+
+services:
+  nginx:
+    image: nginx:1.23
+    depends_on:
+      django:
+        condition: service_healthy
+    volumes:
+      - ./config/nginx/conf.d:/etc/nginx/conf.d
+      - logs:/var/log/nginx

docker-compose.yml

@@ -1,62 +1,84 @@
-version: "3"
+version: '3.6'
+
 volumes:
+  logs:
+  media:
   postgres-data:
-  static-files:
-  node-modules:
 
 services:
   db:
-    image: postgres
+    image: postgres:15
+    healthcheck:
+      test: /usr/bin/pg_isready
+      interval: 5s
+      timeout: 10s
+      retries: 10
     environment:
-      POSTGRES_DB: "newsreader"
+      POSTGRES_DB: 'newsreader'
-      POSTGRES_USER: "newsreader"
+      POSTGRES_USER: 'newsreader'
-      POSTGRES_PASSWORD: "newsreader"
+      POSTGRES_PASSWORD: 'newsreader'
     volumes:
       - postgres-data:/var/lib/postgresql/data
+
   rabbitmq:
     image: rabbitmq:3.7
+
   memcached:
     image: memcached:1.6
     ports:
-      - "11211:11211"
+      - '11211:11211'
     entrypoint:
       - memcached
       - -m 64
+
   celery:
     build:
       context: .
       dockerfile: ./docker/django
-    command: celery worker -n worker1@%h -n worker2@%h --app newsreader --loglevel INFO --concurrency 2 --workdir /app/src/ --beat --scheduler django
+    command: |
+      celery worker -n worker1@%h
+      -n worker2@%h
+      --app newsreader
+      --loglevel INFO
+      --concurrency 2
+      --workdir /app/src/
+      --beat
+      --scheduler django
     environment:
-      - DJANGO_SETTINGS_MODULE=newsreader.conf.docker
+      - DJANGO_SETTINGS_MODULE=newsreader.conf.production
     depends_on:
-      - rabbitmq
+      rabbitmq:
-      - memcached
+        condition: service_started
+      memcached:
+        condition: service_started
+      db:
+        condition: service_healthy
+      django:
+        condition: service_healthy
     volumes:
-      - ./src/:/app/src
+      - logs:/app/logs
+
   django:
     build:
       context: .
       dockerfile: ./docker/django
-    command: python /app/src/manage.py runserver 0.0.0.0:8000
+      target: production
+    entrypoint: /app/bin/docker-entrypoint.sh
+    command: gunicorn --bind 0.0.0.0:8000 --workers 3 newsreader.wsgi:application
+    healthcheck:
+      test: /usr/bin/curl --fail http://django:8000 || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 10
     environment:
-      - DJANGO_SETTINGS_MODULE=newsreader.conf.docker
+      - DJANGO_SETTINGS_MODULE=newsreader.conf.production
     ports:
-      - "8000:8000"
+      - '8000:8000'
     depends_on:
-      - db
+      memcached:
-      - memcached
+        condition: service_started
+      db:
+        condition: service_healthy
     volumes:
-      - ./src:/app/src
+      - logs:/app/logs
-      - static-files:/app/src/newsreader/static
+      - media:/app/media
-    stdin_open: true
-    tty: true
-  webpack:
-    build:
-      context: .
-      dockerfile: ./docker/webpack
-    command: npm run build:watch
-    volumes:
-      - ./src/:/app/src
-      - static-files:/app/src/newsreader/static
-      - node-modules:/app/node_modules

docker/django

@@ -1,10 +1,105 @@
-FROM python:3.9-bullseye
+# stage 1
+FROM python:3.9-bullseye as backend
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        vim \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 RUN mkdir /app/src
+RUN mkdir /app/logs
+RUN mkdir /app/media
 
 COPY ./requirements /app/requirements
 
-RUN pip install -r requirements/production.txt -r requirements/development.txt
+RUN pip install -r requirements/production.txt
+
+
+# stage 2
+FROM node:current-bullseye AS frontend-build
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY ./build /app/build/
+COPY ./*.json ./*.js ./.babelrc /app/
+
+RUN npm ci
 
 COPY ./src /app/src
+
+RUN npm run build
+
+
+# stage 3
+FROM python:3.9-bullseye as production
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        postgresql-client \
+        vim \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+RUN mkdir /app/logs
+RUN mkdir /app/media
+RUN mkdir /app/bin
+
+COPY --from=backend /usr/local/lib/python3.9 /usr/local/lib/python3.9
+COPY --from=backend /usr/local/bin/gunicorn /usr/local/bin/gunicorn
+COPY --from=backend /usr/local/bin/celery /usr/local/bin/celery
+COPY --from=backend /app/src/ /app/src/
+COPY ./bin/docker-entrypoint.sh /app/bin/docker-entrypoint.sh
+
+COPY --from=frontend-build /app/src/newsreader/static /app/src/newsreader/static
+
+COPY ./src /app/src
+
+RUN useradd -M -u 1000 newsreader
+RUN chown -R newsreader:newsreader /app
+
+USER newsreader
+
+ARG COMMIT_HASH
+ARG RELEASE=latest
+
+ENV RELEASE=${RELEASE} \
+    GIT_SHA=${COMMIT_HASH} \
+    PYTHONUNBUFFERED=1 \
+    DJANGO_SETTINGS_MODULE=newsreader.conf.production
+
+ARG SECRET_KEY=dummy
+
+RUN python src/manage.py collectstatic --noinput \
+    && python src/manage.py compilemessages
+
+
+# (optional) stage 4
+FROM python:3.9-bullseye as development
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        vim \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+RUN mkdir /app/bin
+
+COPY ./requirements /app/requirements
+COPY ./bin/docker-entrypoint.sh /app/bin/docker-entrypoint.sh
+COPY --from=backend /usr/local/lib/python3.9 /usr/local/lib/python3.9
+COPY --from=backend /usr/local/bin/celery /usr/local/bin/celery
+COPY --from=backend /app/src/ /app/src/
+
+RUN pip install -r requirements/production.txt -r requirements/development.txt
+
+RUN useradd -M -u 1000 newsreader
+RUN chown -R newsreader:newsreader /app
+
+USER newsreader

gitlab-ci/deploy.yml

@@ -1,22 +0,0 @@
-deploy:
-  stage: deploy
-  image: python:3.7
-  environment:
-    name: production
-    url: rss.fudiggity.nl
-  rules:
-    - if: $CI_COMMIT_TAG
-  before_script:
-    - pip install ansible --quiet
-    - git clone https://git.fudiggity.nl/ansible/newsreader.git deployment --branch master
-    - cd deployment
-    - ansible-galaxy install -r requirements.yml
-    - mkdir /root/.ssh && echo "$DEPLOY_HOST_KEY" > /root/.ssh/known_hosts
-    - echo "$DEPLOY_KEY" > deploy_key && chmod 0600 deploy_key
-    - echo "$VAULT_PASSWORD" > vault
-  script:
-    - >
-      ansible-playbook playbook.yml
-      --private-key deploy_key
-      --vault-password-file vault
-      --extra-vars "app_branch=$CI_COMMIT_TAG"

gitlab-ci/test.yml

@@ -2,7 +2,7 @@ python-tests:
   stage: test
   coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
   services:
-    - postgres:11
+    - postgres:15
     - memcached:1.5.22
   image: python:3.9-bullseye
   before_script:

src/newsreader/conf/base.py

@@ -21,7 +21,7 @@ DJANGO_PROJECT_DIR = os.path.join(BASE_DIR, "src", "newsreader")
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
 # SECURITY WARNING: don"t run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
 INTERNAL_IPS = ["127.0.0.1", "localhost"]
@@ -208,9 +208,6 @@ STATICFILES_FINDERS = [
 # Email
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 
-# Project settings
-ENVIRONMENT = "development"
-
 # Reddit integration
 REDDIT_CLIENT_ID = "CLIENT_ID"
 REDDIT_CLIENT_SECRET = "CLIENT_SECRET"
@@ -261,7 +258,6 @@ ACCOUNT_ACTIVATION_DAYS = 7
 SENTRY_CONFIG = {
     "dsn": os.environ.get("SENTRY_DSN"),
     "send_default_pii": False,
-    "environment": ENVIRONMENT,
     "integrations": [DjangoIntegration(), CeleryIntegration()]
     if DjangoIntegration and CeleryIntegration
     else [],

src/newsreader/conf/docker.py

@@ -2,6 +2,8 @@ from .base import *  # isort:skip
 from .version import get_current_version
 
 
+ALLOWED_HOSTS = ["django", "127.0.0.1"]
+
 SECRET_KEY = "=q(ztyo)b6noom#a164g&s9vcj1aawa^g#ing_ir99=_zl4g&$"
 
 INSTALLED_APPS += ["debug_toolbar", "django_extensions"]
@@ -31,6 +33,8 @@ CACHES = {
     },
 }
 
+DEBUG = True
+
 # Project settings
 VERSION = get_current_version()
 ENVIRONMENT = "docker"