From 5a73707d6150aa3e0a70a2f66b2eb8085a903b8e Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Wed, 28 Jun 2023 19:38:44 +0200 Subject: [PATCH] Split production dependecies & update production configuration --- Makefile | 10 +++++++++- config/nginx/conf.d/local.conf | 4 ++++ docker-compose.production.yml | 4 +++- docker-compose.yml | 22 +++++++++++++++++++++ docker/django | 33 ++++++++++++++++--------------- pyproject.toml | 5 ++--- requirements/base.txt | 14 +++---------- requirements/ci.txt | 31 +---------------------------- requirements/development.txt | 32 ++---------------------------- requirements/production.txt | 10 +++------- requirements/testing.txt | 12 +---------- src/newsreader/conf/production.py | 2 +- 12 files changed, 68 insertions(+), 111 deletions(-) diff --git a/Makefile b/Makefile index ca182ad..4cddf45 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,14 @@ # # Build dependencies build: - pip-compile --output-file=requirements/base.txt pyproject.toml + pip-compile \ + --resolver=backtracking \ + --output-file=requirements/base.txt \ + pyproject.toml # testing pip-compile \ + --resolver=backtracking \ --extra=testing \ --output-file=requirements/testing.txt \ requirements/base.txt \ @@ -14,6 +18,7 @@ build: # development pip-compile \ + --resolver=backtracking \ --extra=testing \ --extra=development \ --output-file=requirements/development.txt \ @@ -23,6 +28,7 @@ build: # ci pip-compile \ + --resolver=backtracking \ --extra=testing \ --extra=ci \ --output-file=requirements/ci.txt \ @@ -32,6 +38,8 @@ build: # production pip-compile \ + --resolver=backtracking \ + --extra=production \ --output-file=requirements/production.txt \ requirements/base.txt \ pyproject.toml diff --git a/config/nginx/conf.d/local.conf b/config/nginx/conf.d/local.conf index 45bba56..b781ca5 100644 --- a/config/nginx/conf.d/local.conf +++ b/config/nginx/conf.d/local.conf @@ -9,6 +9,10 @@ server { access_log /var/log/nginx/access_log; error_log /var/log/nginx/error_log; + location /static/ { + root /app; + } + location / { proxy_pass http://gunicorn; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/docker-compose.production.yml b/docker-compose.production.yml index ac121c5..aa442ad 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -2,6 +2,7 @@ version: "3.6" volumes: logs: + static-files: services: nginx: @@ -9,9 +10,10 @@ services: depends_on: django: condition: service_healthy + env_file: ./.env ports: - "${NGINX_HTTP_PORT:-80}:80" - - "${NGINX_HTTPS_PORT:-443}:443" volumes: - ./config/nginx/conf.d:/etc/nginx/conf.d - logs:/var/log/nginx + - static-files:/app/static diff --git a/docker-compose.yml b/docker-compose.yml index db5b91a..d383081 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ volumes: logs: media: postgres-data: + static-files: services: db: @@ -13,6 +14,7 @@ services: interval: 5s timeout: 10s retries: 10 + env_file: ./.env volumes: - postgres-data:/var/lib/postgresql/data @@ -29,6 +31,15 @@ services: build: context: . dockerfile: ./docker/django + target: production + args: + POSTGRES_HOST: + POSTGRES_PORT: + POSTGRES_DB: + POSTGRES_USER: + POSTGRES_PASSWORD: + DJANGO_SECRET_KEY: + DJANGO_SETTINGS_MODULE: "newsreader.conf.production" command: | celery worker -n worker1@%h -n worker2@%h @@ -57,16 +68,26 @@ services: context: . dockerfile: ./docker/django target: production + args: + POSTGRES_HOST: + POSTGRES_PORT: + POSTGRES_DB: + POSTGRES_USER: + POSTGRES_PASSWORD: + DJANGO_SECRET_KEY: + DJANGO_SETTINGS_MODULE: "newsreader.conf.production" entrypoint: /app/bin/docker-entrypoint.sh command: | gunicorn --bind 0.0.0.0:8000 --workers 3 + --chdir /app/src/ newsreader.wsgi:application healthcheck: test: /usr/bin/curl --fail http://django:8000 || exit 1 interval: 30s timeout: 10s retries: 10 + env_file: ./.env environment: - DJANGO_SETTINGS_MODULE=newsreader.conf.production depends_on: @@ -77,3 +98,4 @@ services: volumes: - logs:/app/logs - media:/app/media + - static-files:/app/static diff --git a/docker/django b/docker/django index b9a1208..18e23af 100644 --- a/docker/django +++ b/docker/django @@ -4,6 +4,7 @@ FROM python:3.9-bullseye as backend RUN apt-get update && apt-get install -y --no-install-recommends \ vim \ curl \ + gettext \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -13,11 +14,11 @@ RUN mkdir /app/media COPY ./requirements /app/requirements -RUN pip install -r requirements/production.txt +RUN pip install -r requirements/base.txt # stage 2 -FROM node:current-bullseye AS frontend-build +FROM node:16-bullseye AS frontend-build RUN apt-get update && apt-get install -y --no-install-recommends \ git \ @@ -25,7 +26,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app -COPY ./build /app/build/ COPY ./*.json ./*.js ./.babelrc /app/ RUN npm ci @@ -42,6 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ postgresql-client \ vim \ curl \ + gettext \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -51,32 +52,32 @@ RUN mkdir /app/media RUN mkdir /app/bin COPY --from=backend /usr/local/lib/python3.9 /usr/local/lib/python3.9 -COPY --from=backend /usr/local/bin/gunicorn /usr/local/bin/gunicorn COPY --from=backend /usr/local/bin/celery /usr/local/bin/celery -COPY --from=backend /app/src/ /app/src/ + COPY ./bin/docker-entrypoint.sh /app/bin/docker-entrypoint.sh COPY --from=frontend-build /app/src/newsreader/static /app/src/newsreader/static COPY ./src /app/src +COPY ./requirements /app/requirements + +RUN pip install -r requirements/production.txt + RUN useradd -M -u 1000 newsreader RUN chown -R newsreader:newsreader /app USER newsreader -ARG COMMIT_HASH -ARG RELEASE=latest +ARG POSTGRES_HOST +ARG POSTGRES_PORT +ARG POSTGRES_DB +ARG POSTGRES_USER +ARG POSTGRES_PASSWORD +ARG DJANGO_SECRET_KEY +ARG DJANGO_SETTINGS_MODULE -ENV RELEASE=${RELEASE} \ - GIT_SHA=${COMMIT_HASH} \ - PYTHONUNBUFFERED=1 \ - DJANGO_SETTINGS_MODULE=newsreader.conf.production - -ARG SECRET_KEY=dummy - -RUN python src/manage.py collectstatic --noinput \ - && python src/manage.py compilemessages +RUN python src/manage.py collectstatic --noinput # (optional) stage 4 diff --git a/pyproject.toml b/pyproject.toml index 66b6434..3ee8c9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,6 @@ requires-python = '>=3.11' dependencies = [ 'django~=3.2', 'celery~=4.4', - 'gunicorn~=20.0', 'psycopg2', 'django-axes', @@ -19,8 +18,6 @@ dependencies = [ 'python-memcached', 'python-dotenv~=0.12', - 'sentry-sdk~=1.0', - 'ftfy~=5.8', 'requests', @@ -49,3 +46,5 @@ development = [ ] ci = ['coverage>=5.3.1'] + +production = ['gunicorn~=20.0', 'sentry-sdk~=1.0'] diff --git a/requirements/base.txt b/requirements/base.txt index ca92e22..1cb598e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --output-file=requirements/base.txt pyproject.toml +# pip-compile --output-file=requirements/base.txt --resolver=backtracking pyproject.toml # amqp==2.6.1 # via kombu @@ -19,9 +19,7 @@ celery==4.4.7 # django-celery-beat # newsreader (pyproject.toml) certifi==2023.5.7 - # via - # requests - # sentry-sdk + # via requests charset-normalizer==3.1.0 # via requests django==3.2.19 @@ -52,8 +50,6 @@ feedparser==6.0.10 # via newsreader (pyproject.toml) ftfy==5.9 # via newsreader (pyproject.toml) -gunicorn==20.1.0 - # via newsreader (pyproject.toml) idna==3.4 # via requests inflection==0.5.1 @@ -91,8 +87,6 @@ requests==2.31.0 # requests-oauthlib requests-oauthlib==1.3.1 # via newsreader (pyproject.toml) -sentry-sdk==1.26.0 - # via newsreader (pyproject.toml) sgmllib3k==1.0.0 # via feedparser six==1.16.0 @@ -109,9 +103,7 @@ typing-extensions==4.6.3 uritemplate==4.1.1 # via drf-yasg urllib3==2.0.3 - # via - # requests - # sentry-sdk + # via requests vine==1.3.0 # via # amqp diff --git a/requirements/ci.txt b/requirements/ci.txt index 5af4236..1d71afc 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --extra=ci --extra=testing --output-file=requirements/ci.txt pyproject.toml requirements/base.txt requirements/testing.txt +# pip-compile --extra=ci --extra=testing --output-file=requirements/ci.txt --resolver=backtracking pyproject.toml requirements/base.txt requirements/testing.txt # amqp==2.6.1 # via @@ -21,7 +21,6 @@ autoflake==2.2.0 beautifulsoup4==4.12.2 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) billiard==3.6.4.0 # via @@ -35,12 +34,10 @@ black==23.3.0 bleach==6.0.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) celery==4.4.7 # via # -r requirements/base.txt - # -r requirements/testing.txt # django-celery-beat # newsreader (pyproject.toml) certifi==2023.5.7 @@ -48,7 +45,6 @@ certifi==2023.5.7 # -r requirements/base.txt # -r requirements/testing.txt # requests - # sentry-sdk charset-normalizer==3.1.0 # via # -r requirements/base.txt @@ -63,7 +59,6 @@ coverage==7.2.7 django==3.2.19 # via # -r requirements/base.txt - # -r requirements/testing.txt # django-axes # django-celery-beat # django-timezone-field @@ -73,22 +68,18 @@ django==3.2.19 django-axes==6.0.4 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-celery-beat==2.0.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-registration-redux==2.12 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-rest-framework==0.1.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-timezone-field==4.2.3 # via @@ -104,7 +95,6 @@ djangorestframework==3.14.0 drf-yasg==1.21.6 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) factory-boy==3.2.1 # via @@ -117,7 +107,6 @@ faker==18.11.2 feedparser==6.0.10 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) freezegun==1.2.2 # via @@ -126,12 +115,6 @@ freezegun==1.2.2 ftfy==5.9 # via # -r requirements/base.txt - # -r requirements/testing.txt - # newsreader (pyproject.toml) -gunicorn==20.1.0 - # via - # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) idna==3.4 # via @@ -155,7 +138,6 @@ kombu==4.6.11 lxml==4.9.2 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) mypy-extensions==1.0.0 # via @@ -183,7 +165,6 @@ platformdirs==3.8.0 psycopg2==2.9.6 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) pyflakes==3.0.1 # via @@ -204,12 +185,10 @@ python-dateutil==2.8.2 python-dotenv==0.21.1 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) python-memcached==1.59 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) pytz==2023.3 # via @@ -228,18 +207,11 @@ pyyaml==6.0 requests==2.31.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) # requests-oauthlib requests-oauthlib==1.3.1 # via # -r requirements/base.txt - # -r requirements/testing.txt - # newsreader (pyproject.toml) -sentry-sdk==1.26.0 - # via - # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) sgmllib3k==1.0.0 # via @@ -288,7 +260,6 @@ urllib3==2.0.3 # -r requirements/base.txt # -r requirements/testing.txt # requests - # sentry-sdk vine==1.3.0 # via # -r requirements/base.txt diff --git a/requirements/development.txt b/requirements/development.txt index 45ca8a4..5f06dd7 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --extra=development --extra=testing --output-file=requirements/development.txt pyproject.toml requirements/base.txt requirements/testing.txt +# pip-compile --extra=development --extra=testing --output-file=requirements/development.txt --resolver=backtracking pyproject.toml requirements/base.txt requirements/testing.txt # amqp==2.6.1 # via @@ -21,7 +21,6 @@ autoflake==2.2.0 beautifulsoup4==4.12.2 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) billiard==3.6.4.0 # via @@ -35,14 +34,12 @@ black==23.3.0 bleach==6.0.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) build==0.10.0 # via pip-tools celery==4.4.7 # via # -r requirements/base.txt - # -r requirements/testing.txt # django-celery-beat # newsreader (pyproject.toml) certifi==2023.5.7 @@ -50,7 +47,6 @@ certifi==2023.5.7 # -r requirements/base.txt # -r requirements/testing.txt # requests - # sentry-sdk charset-normalizer==3.1.0 # via # -r requirements/base.txt @@ -64,7 +60,6 @@ click==8.1.3 django==3.2.19 # via # -r requirements/base.txt - # -r requirements/testing.txt # django-axes # django-celery-beat # django-debug-toolbar @@ -76,12 +71,10 @@ django==3.2.19 django-axes==6.0.4 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-celery-beat==2.0.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-debug-toolbar==4.1.0 # via newsreader (pyproject.toml) @@ -90,12 +83,10 @@ django-extensions==3.2.3 django-registration-redux==2.12 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-rest-framework==0.1.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) django-timezone-field==4.2.3 # via @@ -111,7 +102,6 @@ djangorestframework==3.14.0 drf-yasg==1.21.6 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) factory-boy==3.2.1 # via @@ -124,7 +114,6 @@ faker==18.11.2 feedparser==6.0.10 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) freezegun==1.2.2 # via @@ -133,12 +122,6 @@ freezegun==1.2.2 ftfy==5.9 # via # -r requirements/base.txt - # -r requirements/testing.txt - # newsreader (pyproject.toml) -gunicorn==20.1.0 - # via - # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) idna==3.4 # via @@ -162,7 +145,6 @@ kombu==4.6.11 lxml==4.9.2 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) mypy-extensions==1.0.0 # via @@ -193,7 +175,6 @@ platformdirs==3.8.0 psycopg2==2.9.6 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) pyflakes==3.0.1 # via @@ -216,12 +197,10 @@ python-dateutil==2.8.2 python-dotenv==0.21.1 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) python-memcached==1.59 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) pytz==2023.3 # via @@ -240,18 +219,11 @@ pyyaml==6.0 requests==2.31.0 # via # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) # requests-oauthlib requests-oauthlib==1.3.1 # via # -r requirements/base.txt - # -r requirements/testing.txt - # newsreader (pyproject.toml) -sentry-sdk==1.26.0 - # via - # -r requirements/base.txt - # -r requirements/testing.txt # newsreader (pyproject.toml) sgmllib3k==1.0.0 # via @@ -286,6 +258,7 @@ tomli==2.0.1 # autoflake # black # build + # pyproject-hooks typing-extensions==4.6.3 # via # -r requirements/base.txt @@ -302,7 +275,6 @@ urllib3==2.0.3 # -r requirements/base.txt # -r requirements/testing.txt # requests - # sentry-sdk vine==1.3.0 # via # -r requirements/base.txt diff --git a/requirements/production.txt b/requirements/production.txt index e9a69fa..d86aa4f 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --output-file=requirements/production.txt pyproject.toml requirements/base.txt +# pip-compile --extra=production --output-file=requirements/production.txt --resolver=backtracking pyproject.toml requirements/base.txt # amqp==2.6.1 # via @@ -85,9 +85,7 @@ ftfy==5.9 # -r requirements/base.txt # newsreader (pyproject.toml) gunicorn==20.1.0 - # via - # -r requirements/base.txt - # newsreader (pyproject.toml) + # via newsreader (pyproject.toml) idna==3.4 # via # -r requirements/base.txt @@ -154,9 +152,7 @@ requests-oauthlib==1.3.1 # -r requirements/base.txt # newsreader (pyproject.toml) sentry-sdk==1.26.0 - # via - # -r requirements/base.txt - # newsreader (pyproject.toml) + # via newsreader (pyproject.toml) sgmllib3k==1.0.0 # via # -r requirements/base.txt diff --git a/requirements/testing.txt b/requirements/testing.txt index e6bb278..820d99d 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --extra=testing --output-file=requirements/testing.txt pyproject.toml requirements/base.txt +# pip-compile --extra=testing --output-file=requirements/testing.txt --resolver=backtracking pyproject.toml requirements/base.txt # amqp==2.6.1 # via @@ -37,7 +37,6 @@ certifi==2023.5.7 # via # -r requirements/base.txt # requests - # sentry-sdk charset-normalizer==3.1.0 # via # -r requirements/base.txt @@ -96,10 +95,6 @@ ftfy==5.9 # via # -r requirements/base.txt # newsreader (pyproject.toml) -gunicorn==20.1.0 - # via - # -r requirements/base.txt - # newsreader (pyproject.toml) idna==3.4 # via # -r requirements/base.txt @@ -178,10 +173,6 @@ requests-oauthlib==1.3.1 # via # -r requirements/base.txt # newsreader (pyproject.toml) -sentry-sdk==1.26.0 - # via - # -r requirements/base.txt - # newsreader (pyproject.toml) sgmllib3k==1.0.0 # via # -r requirements/base.txt @@ -219,7 +210,6 @@ urllib3==2.0.3 # via # -r requirements/base.txt # requests - # sentry-sdk vine==1.3.0 # via # -r requirements/base.txt diff --git a/src/newsreader/conf/production.py b/src/newsreader/conf/production.py index cdc0152..31900e9 100644 --- a/src/newsreader/conf/production.py +++ b/src/newsreader/conf/production.py @@ -7,7 +7,7 @@ from .base import * # isort:skip DEBUG = False -ALLOWED_HOSTS = ["rss.fudiggity.nl"] +ALLOWED_HOSTS = ["127.0.0.1", "localhost", "rss.fudiggity.nl", "django"] ADMINS = [ ("", email) for email in os.getenv("ADMINS", "").split(",")