commit 5fe082b8f6032b6cdb0f70c10d2e6e21a6def870 Author: Sonny Bakker Date: Sat Nov 7 21:52:02 2020 +0100 Initial commit diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..e99d805 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,5 @@ +parseable: true +quiet: true +skip_list: + - '501' +use_default_rules: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c17815f --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +*.retry +*.swp + +.venv +.env +env +venv + +node_modules/ + +.vault +.vaults/ +vault +vaults/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..a4dfde9 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,41 @@ +stages: + - lint + - test + +cache: + key: "$CI_COMMIT_REF_SLUG" + paths: + - .cache/pip + - node_modules/ + +lint: + stage: lint + image: python:3.7 + before_script: + - pip install ansible ansible-lint --quiet + script: + - ansible-lint playbook.yml + only: + refs: + - development + - merge_requests + +pretty-lint: + stage: lint + image: node:12 + before_script: + - npm install + script: + - npx prettier "**/*.yml" --check + only: + refs: + - development + - merge_requests + +syntax-test: + stage: test + image: python:3.7 + before_script: + - pip install ansible ansible-lint --quiet + script: + - ansible-playbook playbook.yml --syntax-check diff --git a/.prettier.json b/.prettier.json new file mode 100644 index 0000000..9c76f6b --- /dev/null +++ b/.prettier.json @@ -0,0 +1,9 @@ +{ + "singleQuote": true, + "printWidth": 90, + "tabWidth": 2, + "useTabs": false, + "bracketSpacing": true, + "parser": "yaml" +} + diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..002a50d --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +roles_path = ./roles diff --git a/inventory.yml b/inventory.yml new file mode 100644 index 0000000..5a7fd86 --- /dev/null +++ b/inventory.yml @@ -0,0 +1,3 @@ +newsreader: + hosts: + 192.168.178.63: diff --git a/playbook.yml b/playbook.yml new file mode 100644 index 0000000..6cb8595 --- /dev/null +++ b/playbook.yml @@ -0,0 +1,5 @@ +- hosts: newsreader + become: yes + become_method: sudo + roles: + - newsreader diff --git a/roles/.gitignore b/roles/.gitignore new file mode 100644 index 0000000..b9b8935 --- /dev/null +++ b/roles/.gitignore @@ -0,0 +1,6 @@ +# ignore all external roles and files in the roles dir +/* + +!.gitignore +!requirements.yml +!newsreader*/ diff --git a/roles/newsreader/defaults/main/app.yml b/roles/newsreader/defaults/main/app.yml new file mode 100644 index 0000000..266ade4 --- /dev/null +++ b/roles/newsreader/defaults/main/app.yml @@ -0,0 +1,18 @@ +app_name: "newsreader" +app_user: "newsreader" +app_branch: "master" +app_dir: "/srv/sites/newsreader" +app_deploy_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICq4U2MKiufVeM8fVzpfoA/rhiWTMnrJr9usAZAG6kfb Key for deploying to newsreader app" + +django_settings_module: "newsreader.conf.production" +django_secret_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 37383464313435333061393165373731303161343236666138313566333631303839393163313038 + 3934316466383964313962373865393164393164363332330a326333313631636132313033376230 + 66653634666463393061383731303661643662653036316332663039396164363432386137336135 + 3339336563316434330a376233333762656162323139336535366136633866626532376662663635 + 62656331306464363637393164633535393339613834383036646262326539393638393532633038 + 35626539383762383462646632616334633737623035643034643433623237323932373334316639 + 356533316361653939303165313766633666 + +admins: "" diff --git a/roles/newsreader/defaults/main/main.yml b/roles/newsreader/defaults/main/main.yml new file mode 100644 index 0000000..c803205 --- /dev/null +++ b/roles/newsreader/defaults/main/main.yml @@ -0,0 +1,14 @@ +default_user: "sonny" + +gitlab_host_key: "git.fudiggity.nl ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICNmvcxza79T7JZMkifmquwXH/kMUqDnKs9Oob+JrRvn" +gitlab_domain: "git.fudiggity.nl" + +sentry_dsn: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 62663633303937303836306431396435343838626439373733353464366266383364343365333735 + 3539306162396230633664663839333637303733323461620a376531366464646239366436343136 + 30643031653363616265363938633536386134343766656239376638643335396565326330613832 + 6639396631623334350a353037386433313166313736666161663964313932636462383461343463 + 64636433373533313933343461393638663638623430366332336265343061663030313031353665 + 63393062396534643934623964323437316238353535623261323531383230633536626333326262 + 656365343030643332303534343535336566 diff --git a/roles/newsreader/defaults/main/network.yml b/roles/newsreader/defaults/main/network.yml new file mode 100644 index 0000000..9985a99 --- /dev/null +++ b/roles/newsreader/defaults/main/network.yml @@ -0,0 +1,6 @@ +hostname: "rss.fudiggity.nl" +host_interface: "en*" +host_ip: "192.168.178.63" +host_subnet: "24" +host_gateway: "192.168.178.1" +host_dns: "192.168.178.1" diff --git a/roles/newsreader/defaults/main/postgres.yml b/roles/newsreader/defaults/main/postgres.yml new file mode 100644 index 0000000..8247e37 --- /dev/null +++ b/roles/newsreader/defaults/main/postgres.yml @@ -0,0 +1,25 @@ +postgres_host: "192.168.178.165" +postgres_port: "5432" +postgres_db: "newsreader" +postgres_user: "newsreader" +postgres_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32613132353864633832306363626432343063616433343830623064326166653231313334636463 + 6631343261613137363864336435313664653738383232330a356163633637393433313532303563 + 62356162303438323139616338316130373364383331663437336662356432623136396364396539 + 3236326566393034650a613133623062333862623931353131663731623764393632643639363434 + 38623566363265373230386535303332363564393234636561663761353235303132373865353530 + 6138663238346363383737633133383638383962386236343565 + +pgbouncer_listen_address: "127.0.0.1" +pgbouncer_port: "6432" +pgbouncer_name: "newsreader" +pgbouncer_user: "newsreader" +pgbouncer_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 38613333336663643735396637373664363432343633653161633463373536653964656266356564 + 3732356130316365323466626232373835623266363433370a313732613535303864356266303462 + 35333164343062623362353566336439326661633135366238313235626162343165653930383562 + 6431666462643064310a656230623365616334613139363033626463623063313065633462663061 + 34303265643662363034653230323939313664623364633532626266646662643130346665653733 + 6138366237333461343561353333663761303039383261356535 diff --git a/roles/newsreader/defaults/main/reddit.yml b/roles/newsreader/defaults/main/reddit.yml new file mode 100644 index 0000000..222fc80 --- /dev/null +++ b/roles/newsreader/defaults/main/reddit.yml @@ -0,0 +1,15 @@ +reddit_client_id: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 66373431643364616463393031393361393339363335313466353935386536373738373033373433 + 6462633333386430373833623032616266666165313138380a353463383831336565653633666536 + 37666334373766666164343030363335636438363834333339323163316235326461323938633164 + 3465613231336438360a366339633339323362363033303634333566303763643736393634643462 + 6365 +reddit_client_secret: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 62633339373264626134626266363064623765356438666463343934356361653330336162363831 + 6239383363663737623834626230623733303931306635640a633031303837303731346334656438 + 36636464353761383464343634323035666163353561383231623337343732326263353535656165 + 3738633565396265320a343330623938356631376664326562353437333263386538356438653336 + 64326363666638306337386266653331633938316639383034376464306238613839 +reddit_callback_url: "https://rss.fudiggity.nl/accounts/settings/integrations/reddit/callback/" diff --git a/roles/newsreader/defaults/main/twitter.yml b/roles/newsreader/defaults/main/twitter.yml new file mode 100644 index 0000000..bc47204 --- /dev/null +++ b/roles/newsreader/defaults/main/twitter.yml @@ -0,0 +1,17 @@ +twitter_client_id: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 62323232643762313964343763363666363336653534653433663333313732643836366131383830 + 3861373534643464653166383935376165313730323065630a623862663531306162653036636465 + 66306236346366653539643330356630613432373962323730306432643635653234343661376664 + 3864356432343465390a613865666362303630613136363263363066303662646238393961636131 + 63313034653936373531373263613833656665613263353261393066393835663861 +twitter_client_secret: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32316631353336633338303863356633393263303030393533316533383262376233303262386635 + 3136373434636231373237333463666431356239343661660a343762623966326464306538363030 + 36303763373366333034633964656537623635356135623666363031333738396164636535363733 + 3061343530333139610a633037343062373231363831663237376162646535623762343736343165 + 32613037316134643965353138643236636632623865636632363964666161303330336136626264 + 63366438343633653566313231633739343036663736333037353465353439346135663733363137 + 386165313662356630643164396563316562 +twitter_redirect_url: "https://rss.fudiggity.nl/accounts/settings/integrations/twitter/callback/" diff --git a/roles/newsreader/handlers/main.yml b/roles/newsreader/handlers/main.yml new file mode 100644 index 0000000..6ed4827 --- /dev/null +++ b/roles/newsreader/handlers/main.yml @@ -0,0 +1,62 @@ +- name: install npm packages + become_user: "{{ app_user }}" + command: /usr/bin/npm install + args: + chdir: "{{ app_dir }}" + +- name: build static files + become_user: "{{ app_user }}" + command: /usr/bin/npm run build:prod + args: + chdir: "{{ app_dir }}" + +- name: run migrations + become_user: "{{ app_user }}" + django_manage: + command: migrate + app_path: "{{ app_dir }}/src/" + virtualenv: "{{ app_dir }}/.venv" + settings: "newsreader.conf.production" + +- name: collect static files + become_user: "{{ app_user }}" + django_manage: + command: collectstatic + app_path: "{{ app_dir }}/src/" + virtualenv: "{{ app_dir }}/.venv" + settings: "newsreader.conf.production" + +- name: restart gunicorn socket + systemd: + daemon-reload: yes + name: gunicorn.socket + state: restarted + enabled: yes + +- name: stop gunicorn service + systemd: + daemon-reload: yes + name: gunicorn.service + state: stopped + enabled: no + +- name: restart pgbouncer + systemd: + daemon-reload: yes + name: pgbouncer + state: restarted + enabled: yes + +- name: restart celery + systemd: + daemon-reload: yes + name: celery + state: restarted + enabled: yes + +- name: restart celerybeat + systemd: + daemon-reload: yes + name: celerybeat + state: restarted + enabled: yes diff --git a/roles/newsreader/meta/main.yml b/roles/newsreader/meta/main.yml new file mode 100644 index 0000000..76412e0 --- /dev/null +++ b/roles/newsreader/meta/main.yml @@ -0,0 +1,17 @@ +dependencies: + - common + - npm + +galaxy_info: + author: sonny + description: "Newsreader installation" + license: "license GPLv3" + min_ansible_version: 2.7 + issue_tracker_url: "https://git.fudiggity.nl/sonny/ansible-playbooks/-/issues" + platforms: + - name: Debian + versions: + - buster + galaxy_tags: + - development + - web diff --git a/roles/newsreader/tasks/main.yml b/roles/newsreader/tasks/main.yml new file mode 100644 index 0000000..d642c48 --- /dev/null +++ b/roles/newsreader/tasks/main.yml @@ -0,0 +1,168 @@ +- include_role: + name: common + tasks_from: "network.yml" +- include_role: + name: common + tasks_from: "host.yml" +- include_role: + name: common + tasks_from: "sudoers.yml" + loop: + - { + src: "../newsreader/templates/sudoers.j2", + dest: "/etc/sudoers.d/30-ansible-extra", + } + +- name: install packages + apt: + name: + - memcached + - pgbouncer + - postfix + - python-psycopg2 + - python3-psycopg2 + - python3 + - python3-pip + - python3-venv + - python3-setuptools + - python3-virtualenv + - python-pip + - python-setuptools + - python-virtualenv + - rabbitmq-server + state: present + notify: + - restart postfix + +- name: copy firewall templates + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "0600" + with_items: + - { src: "nftables.j2", dest: "/etc/nftables.conf" } + notify: restart nftables + +- name: copy memcached conf + template: + src: "memcached.j2" + dest: "/etc/memcached.conf" + owner: root + group: root + mode: "0644" + notify: restart memcached + +- name: add gitlab to known hosts + become_user: "{{ app_user }}" + known_hosts: + name: "{{ gitlab_domain }}" + key: "{{ gitlab_host_key }}" + +- name: add gitlab pubkey + authorized_key: + user: ansible + state: present + key: "{{ app_deploy_key }}" + +- name: Add newsreader user + user: + name: "{{ app_user }}" + create_home: yes + shell: /bin/bash + +- name: create ssh dir + file: + path: "/home/{{ app_user }}/.ssh" + state: directory + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: 0755 + +- name: create rabbitmq service override dir + file: + path: /etc/systemd/system/rabbitmq-server.service.d/ + state: directory + mode: "0644" + +- name: copy rabbitmq configurations + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: + - { + src: "limits.j2", + dest: "/etc/systemd/system/rabbitmq-server.service.d/limits.conf", + mode: "0644", + group: "root", + owner: "root", + } + - { + src: "rabbitmq.conf.j2", + dest: "/etc/rabbitmq/rabbitmq-env.conf", + mode: "0644", + group: "rabbitmq", + owner: "rabbitmq", + } + notify: restart rabbitmq + +- include_role: + name: common + tasks_from: "ssl.yml" +- include_role: + name: common + tasks_from: "nginx.yml" + +- name: copy nginx config + template: + src: "nginx.j2" + dest: "/etc/nginx/sites-available/newsreader" + owner: root + group: root + mode: "0644" + +- name: link nginx config + file: + src: "/etc/nginx/sites-available/newsreader" + dest: "/etc/nginx/sites-enabled/newsreader" + owner: root + group: root + mode: "0777" + state: link + +- name: copy nftables config + template: + src: "nftables.j2" + dest: "/etc/nftables.conf" + owner: root + group: root + mode: "0600" + notify: restart nftables + +- name: copy pgbouncer config + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: postgres + group: postgres + mode: "{{ item.mode }}" + loop: + - { + src: "pgbouncer.j2", + dest: "/etc/pgbouncer/pgbouncer.ini", + "mode": "0640", + } + - { + src: "pgbouncer-users.j2", + dest: "/etc/pgbouncer/userlist.txt", + "mode": "0640", + } + +- name: ensure pgbouncer is restarted + service: name=pgbouncer state=restarted enabled=yes + +- include_tasks: "project.yml" diff --git a/roles/newsreader/tasks/project.yml b/roles/newsreader/tasks/project.yml new file mode 100644 index 0000000..8bf8bfa --- /dev/null +++ b/roles/newsreader/tasks/project.yml @@ -0,0 +1,138 @@ +- name: create sites dir + file: + path: /srv/sites + state: directory + owner: root + group: root + mode: 0755 + +- name: create project dir + file: + path: "{{ app_dir }}" + state: directory + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: 0755 + +- name: clone project + become_user: "{{ app_user }}" + git: + repo: "https://git.fudiggity.nl/sonny/newsreader.git" + dest: "{{ app_dir }}" + version: "{{ app_branch }}" + update: yes + force: yes + notify: + - install npm packages + - build static files + - run migrations + - collect static files + +- name: include poetry tasks + include_role: + name: common + tasks_from: "poetry.yml" + vars: + poetry_user: "{{ app_user }}" + poetry_dir: "/home/{{ app_user }}/.poetry" + +- name: run poetry tasks + block: + - name: retrieve user $PATH + shell: "echo $PATH" + become_user: "{{ app_user }}" + register: path_stats + + - name: set poetry user variables + set_fact: + poetry_user_path: "{{ path_stats.stdout }}" + + - name: set default venv python version + become_user: "{{ app_user }}" + command: "poetry env use python3.7" + args: + chdir: "{{ app_dir }}" + environment: + PATH: "/home/{{ app_user }}/.local/bin:{{ poetry_user_path }}" + + - name: install project dependencies + become_user: "{{ app_user }}" + command: "poetry install" # noqa 301 + args: + chdir: "{{ app_dir }}" + environment: + PATH: "/home/{{ app_user }}/.local/bin:{{ poetry_user_path }}" + +- name: setup env file + template: + src: "env.j2" + dest: "{{ app_dir }}/.env" + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: 0600 + +- name: setup gunicorn service + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { + src: "gunicorn-socket.j2", + dest: "/etc/systemd/system/gunicorn.socket", + "mode": "0644", + } + - { + src: "gunicorn.j2", + dest: "/etc/systemd/system/gunicorn.service", + "mode": "0644", + } + notify: + - restart gunicorn socket + - stop gunicorn service + +- name: create conf dir + become_user: "{{ app_user }}" + file: + path: "/home/{{ app_user }}/.config/conf.d" + state: directory + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: 0750 + +- name: create celery run dir + file: + path: /run/celery + state: directory + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: 0755 + +- name: copy celery config + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ app_user }}" + group: "{{ app_user }}" + mode: "{{ item.mode }}" + loop: + - { + src: "celery.j2", + dest: "/etc/systemd/system/celery.service", + "mode": "0644", + } + - { + src: "celerybeat.j2", + dest: "/etc/systemd/system/celerybeat.service", + "mode": "0644", + } + - { + src: "celery.env.j2", + dest: "/home/newsreader/.config/conf.d/celery", + "mode": "0640", + } + notify: + - restart celery + - restart celerybeat diff --git a/roles/newsreader/templates/celery.env.j2 b/roles/newsreader/templates/celery.env.j2 new file mode 100644 index 0000000..7a2fe3d --- /dev/null +++ b/roles/newsreader/templates/celery.env.j2 @@ -0,0 +1,30 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +# Name of nodes to start +CELERYD_NODES="worker1 worker2" + +CELERY_BIN="{{ app_dir }}/.venv/bin/celery" + +CELERY_APP="newsreader" + +# The scheduler to be used. +# See https://docs.celeryproject.org/en/stable/userguide/configuration.html#beat-scheduler +CELERY_SCHEDULER="django_celery_beat.schedulers:DatabaseScheduler" + +# How to call manage.py +CELERYD_MULTI="multi" + +# Extra command-line arguments to the worker +CELERYD_OPTS="--time-limit=300 --concurrency=8" + +# - %I will be replaced with the current child process index +# and is important when using the prefork pool to avoid race conditions. +CELERYD_PID_FILE="/run/celery/%n.pid" + +CELERYD_LOG_LEVEL="INFO" +CELERYD_LOG_FILE="/dev/null" + +# you may wish to add these options for Celery Beat +CELERYBEAT_PID_FILE="/run/celery/beat.pid" + +DJANGO_SETTINGS_MODULE="newsreader.conf.production" diff --git a/roles/newsreader/templates/celery.j2 b/roles/newsreader/templates/celery.j2 new file mode 100644 index 0000000..4ded4d8 --- /dev/null +++ b/roles/newsreader/templates/celery.j2 @@ -0,0 +1,25 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +[Unit] +Description=Celery Service +After=systemd-networkd-wait-online.service + +[Service] +Type=forking +User=newsreader +Group=newsreader +SyslogIdentifier=celery +EnvironmentFile=/home/newsreader/.config/conf.d/celery +RuntimeDirectory=celery +WorkingDirectory={{ app_dir }}/src +ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --loglevel=${CELERYD_LOG_LEVEL} --logfile=${CELERYD_LOG_FILE} ${CELERYD_OPTS}' +ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ + --pidfile=${CELERYD_PID_FILE}' +ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --loglevel=${CELERYD_LOG_LEVEL} --logfile=${CELERYD_LOG_FILE} ${CELERYD_OPTS}' + +[Install] +WantedBy=multi-user.target diff --git a/roles/newsreader/templates/celerybeat.j2 b/roles/newsreader/templates/celerybeat.j2 new file mode 100644 index 0000000..15c1aec --- /dev/null +++ b/roles/newsreader/templates/celerybeat.j2 @@ -0,0 +1,19 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +[Unit] +Description=Celery Beat Service +After=celery.service + +[Service] +Type=simple +User=newsreader +Group=newsreader +EnvironmentFile=/home/newsreader/.config/conf.d/celery +RuntimeDirectory=celery +WorkingDirectory={{ app_dir }}/src +ExecStart=/bin/sh -c '${CELERY_BIN} beat \ + -A ${CELERY_APP} -S ${CELERY_SCHEDULER} --pidfile=${CELERYBEAT_PID_FILE} \ + --loglevel=${CELERYD_LOG_LEVEL} --logfile=${CELERYD_LOG_FILE}' + +[Install] +WantedBy=multi-user.target diff --git a/roles/newsreader/templates/env.j2 b/roles/newsreader/templates/env.j2 new file mode 100644 index 0000000..3996d5b --- /dev/null +++ b/roles/newsreader/templates/env.j2 @@ -0,0 +1,19 @@ +POSTGRES_HOST="{{ pgbouncer_listen_address }}" +POSTGRES_PORT="{{ pgbouncer_port }}" +POSTGRES_NAME="{{ pgbouncer_name }}" +POSTGRES_USER="{{ pgbouncer_user }}" +POSTGRES_PASSWORD="{{ pgbouncer_password }}" + +DJANGO_SETTINGS_MODULE="{{ django_settings_module }}" +DJANGO_SECRET_KEY="{{ django_secret_key }}" + +REDDIT_CLIENT_ID="{{ reddit_client_id }}" +REDDIT_CLIENT_SECRET="{{ reddit_client_secret }}" +REDDIT_CALLBACK_URL="{{ reddit_callback_url }}" + +TWITTER_CONSUMER_ID="{{ twitter_client_id }}" +TWITTER_CONSUMER_SECRET="{{ twitter_client_secret }}" +TWITTER_REDIRECT_URL="{{ twitter_redirect_url }}" + +SENTRY_DSN="{{ sentry_dsn }}" +ADMINS="{{ admins }}" diff --git a/roles/newsreader/templates/gunicorn-socket.j2 b/roles/newsreader/templates/gunicorn-socket.j2 new file mode 100644 index 0000000..9b55308 --- /dev/null +++ b/roles/newsreader/templates/gunicorn-socket.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +[Unit] +Description=Gunicorn socket + +[Socket] +ListenStream=/run/gunicorn.sock +User=www-data + +[Install] +WantedBy=sockets.target diff --git a/roles/newsreader/templates/gunicorn.j2 b/roles/newsreader/templates/gunicorn.j2 new file mode 100644 index 0000000..b41d3cb --- /dev/null +++ b/roles/newsreader/templates/gunicorn.j2 @@ -0,0 +1,19 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +[Unit] +Description=Gunicorn daemon +Requires=gunicorn.socket +After=network.target + +[Service] +User=newsreader +Group=www-data +EnvironmentFile={{ app_dir }}/.env +WorkingDirectory={{ app_dir }}/src +ExecStart={{ app_dir }}/.venv/bin/gunicorn \ + --workers 3 \ + --bind unix:/run/gunicorn.sock \ + newsreader.wsgi:application + +[Install] +WantedBy=multi-user.target diff --git a/roles/newsreader/templates/limits.j2 b/roles/newsreader/templates/limits.j2 new file mode 100644 index 0000000..a169a80 --- /dev/null +++ b/roles/newsreader/templates/limits.j2 @@ -0,0 +1,6 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +# see https://www.rabbitmq.com/install-debian.html#kernel-resource-limits +# +[Service] +LimitNOFILE=64000 diff --git a/roles/newsreader/templates/memcached.j2 b/roles/newsreader/templates/memcached.j2 new file mode 100644 index 0000000..5abf5d5 --- /dev/null +++ b/roles/newsreader/templates/memcached.j2 @@ -0,0 +1,52 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +# memcached default config file +# 2003 - Jay Bonci +# This configuration file is read by the start-memcached script provided as +# part of the Debian GNU/Linux distribution. + +# Run memcached as a daemon. This command is implied, and is not needed for the +# daemon to run. See the README.Debian that comes with this package for more +# information. +-d + +# Log memcached's output to /var/log/memcached +logfile /var/log/memcached.log + +# Be verbose +# -v + +# Be even more verbose (print client commands as well) +# -vv + +# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default +# Note that the daemon will grow to this size, but does not start out holding this much +# memory +-m 64 + +# Default connection port is 11211 +-p 11211 + +# Run the daemon as root. The start-memcached will default to running as root if no +# -u command is present in this config file +-u memcache + +# Specify which IP address to listen on. The default is to listen on all IP addresses +# This parameter is one of the only security measures that memcached has, so make sure +# it's listening on a firewalled interface. +-l 127.0.0.1 + +# Limit the number of simultaneous incoming connections. The daemon default is 1024 +# -c 1024 + +# Lock down all paged memory. Consult with the README and homepage before you do this +# -k + +# Return error when memory is exhausted (rather than removing items) +# -M + +# Maximize core file limit +# -r + +# Use a pidfile +-P /var/run/memcached/memcached.pid diff --git a/roles/newsreader/templates/nftables.j2 b/roles/newsreader/templates/nftables.j2 new file mode 100644 index 0000000..a7a2257 --- /dev/null +++ b/roles/newsreader/templates/nftables.j2 @@ -0,0 +1,19 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +# vim:set ts=2 sw=2 et: + +flush ruleset + +table inet filter { + chain input { + type filter hook input priority 0; policy drop; + + # accept any localhost traffic + iif lo accept + + # accept traffic originated from us + ct state { established, related } accept + + tcp dport { 22, 80, 443 } accept + } +} diff --git a/roles/newsreader/templates/nginx.j2 b/roles/newsreader/templates/nginx.j2 new file mode 100644 index 0000000..07bc3d0 --- /dev/null +++ b/roles/newsreader/templates/nginx.j2 @@ -0,0 +1,30 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +server { + listen 80; + server_name {{ hostname }}; + return 301 https://$server_name$request_uri; +} + +server { + listen 443 ssl; + server_name {{ hostname }}; + + ssl_certificate /etc/ssl/{{ app_name }}/{{ app_name }}.crt; + ssl_certificate_key /etc/ssl/{{ app_name }}/local.pem; + + access_log /var/log/nginx/{{ app_name }}.log; + error_log /var/log/nginx/{{ app_name }}.log; + + location /static/ { + root /srv/sites/newsreader; + } + + location / { + include proxy_params; + + proxy_redirect off; + + proxy_pass http://unix:/run/gunicorn.sock; + } +} diff --git a/roles/newsreader/templates/pgbouncer-users.j2 b/roles/newsreader/templates/pgbouncer-users.j2 new file mode 100644 index 0000000..9ebef1b --- /dev/null +++ b/roles/newsreader/templates/pgbouncer-users.j2 @@ -0,0 +1 @@ +"{{ pgbouncer_user }}" "{{ pgbouncer_password }}" diff --git a/roles/newsreader/templates/pgbouncer.j2 b/roles/newsreader/templates/pgbouncer.j2 new file mode 100644 index 0000000..532dae5 --- /dev/null +++ b/roles/newsreader/templates/pgbouncer.j2 @@ -0,0 +1,352 @@ +;; {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +;; +;; database name = connect string +;; +;; connect string params: +;; dbname= host= port= user= password= +;; client_encoding= datestyle= timezone= +;; pool_size= connect_query= +;; auth_user= +[databases] +newsreader = host={{ postgres_host }} port={{ postgres_port }} dbname={{ postgres_db }} user={{ postgres_user }} password={{ postgres_password }} + +; foodb over Unix socket +;foodb = + +; redirect bardb to bazdb on localhost +;bardb = host=localhost dbname=bazdb + +; access to dest database will go with single user +;forcedb = host=127.0.0.1 port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' + +; use custom pool sizes +;nondefaultdb = pool_size=50 reserve_pool=10 + +; use auth_user with auth_query if user not present in auth_file +; auth_user must exist in auth_file +; foodb = auth_user=bar + +; fallback connect string +;* = host=testserver + +;; Configuration section +[pgbouncer] + +;;; +;;; Administrative settings +;;; + +logfile = /var/log/postgresql/pgbouncer.log +pidfile = /var/run/postgresql/pgbouncer.pid + +;;; +;;; Where to wait for clients +;;; + +; IP address or * which means all IPs +listen_addr = {{ pgbouncer_listen_address }} +listen_port = {{ pgbouncer_port }} + +; Unix socket is also used for -R. +; On Debian it should be /var/run/postgresql +;unix_socket_dir = /tmp +;unix_socket_mode = 0777 +;unix_socket_group = +unix_socket_dir = /var/run/postgresql + +;;; +;;; TLS settings for accepting clients +;;; + +;; disable, allow, require, verify-ca, verify-full +;client_tls_sslmode = disable + +;; Path to file that contains trusted CA certs +;client_tls_ca_file = + +;; Private key and cert to present to clients. +;; Required for accepting TLS connections from clients. +;client_tls_key_file = +;client_tls_cert_file = + +;; fast, normal, secure, legacy, +;client_tls_ciphers = fast + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2 +;client_tls_protocols = all + +;; none, auto, legacy +;client_tls_dheparams = auto + +;; none, auto, +;client_tls_ecdhcurve = auto + +;;; +;;; TLS settings for connecting to backend databases +;;; + +;; disable, allow, require, verify-ca, verify-full +server_tls_sslmode = require + +;; Path to that contains trusted CA certs +;server_tls_ca_file = + +;; Private key and cert to present to backend. +;; Needed only if backend server require client cert. +;server_tls_key_file = +;server_tls_cert_file = + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2 +server_tls_protocols = secure + +;; fast, normal, secure, legacy, +;server_tls_ciphers = fast + +;;; +;;; Authentication settings +;;; + +; any, trust, plain, crypt, md5, cert, hba, pam +auth_type = trust +auth_file = /etc/pgbouncer/userlist.txt + +;; Path to HBA-style auth config +;auth_hba_file = + +;; Query to use to fetch password from database. Result +;; must have 2 columns - username and password hash. +;auth_query = SELECT usename, passwd FROM pg_shadow WHERE usename=$1 + +;;; +;;; Users allowed into database 'pgbouncer' +;;; + +; comma-separated list of users, who are allowed to change settings +;admin_users = user2, someadmin, otheradmin + +; comma-separated list of users who are just allowed to use SHOW command +;stats_users = stats, root + +;;; +;;; Pooler personality questions +;;; + +; When server connection is released back to pool: +; session - after client disconnects +; transaction - after transaction finishes +; statement - after statement finishes +pool_mode = session + +; +; Query for cleaning connection immediately after releasing from client. +; No need to put ROLLBACK here, pgbouncer does not reuse connections +; where transaction is left open. +; +; Query for 8.3+: +; DISCARD ALL; +; +; Older versions: +; RESET ALL; SET SESSION AUTHORIZATION DEFAULT +; +; Empty if transaction pooling is in use. +; +server_reset_query = DISCARD ALL + + +; Whether server_reset_query should run in all pooling modes. +; If it is off, server_reset_query is used only for session-pooling. +;server_reset_query_always = 0 + +; +; Comma-separated list of parameters to ignore when given +; in startup packet. Newer JDBC versions require the +; extra_float_digits here. +; +;ignore_startup_parameters = extra_float_digits + +; +; When taking idle server into use, this query is ran first. +; SELECT 1 +; +;server_check_query = select 1 + +; If server was used more recently that this many seconds ago, +; skip the check query. Value 0 may or may not run in immediately. +;server_check_delay = 30 + +; Close servers in session pooling mode after a RECONNECT, RELOAD, +; etc. when they are idle instead of at the end of the session. +;server_fast_close = 0 + +;; Use as application_name on server. +;application_name_add_host = 0 + +;;; +;;; Connection limits +;;; + +; total number of clients that can connect +max_client_conn = 100 + +; default pool size. 20 is good number when transaction pooling +; is in use, in session pooling it needs to be the number of +; max clients you want to handle at any moment +default_pool_size = 20 + +;; Minimum number of server connections to keep in pool. +;min_pool_size = 0 + +; how many additional connection to allow in case of trouble +;reserve_pool_size = 0 + +; if a clients needs to wait more than this many seconds, use reserve pool +;reserve_pool_timeout = 5 + +; how many total connections to a single database to allow from all pools +;max_db_connections = 0 +;max_user_connections = 0 + +; If off, then server connections are reused in LIFO manner +;server_round_robin = 0 + +;;; +;;; Logging +;;; + +;; Syslog settings +;syslog = 0 +;syslog_facility = daemon +;syslog_ident = pgbouncer + +; log if client connects or server connection is made +;log_connections = 1 + +; log if and why connection was closed +;log_disconnections = 1 + +; log error messages pooler sends to clients +;log_pooler_errors = 1 + +;; Period for writing aggregated stats into log. +;stats_period = 60 + +;; Logging verbosity. Same as -v switch on command line. +;verbose = 0 + +;;; +;;; Timeouts +;;; + +;; Close server connection if its been connected longer. +;server_lifetime = 3600 + +;; Close server connection if its not been used in this time. +;; Allows to clean unnecessary connections from pool after peak. +;server_idle_timeout = 600 + +;; Cancel connection attempt if server does not answer takes longer. +;server_connect_timeout = 15 + +;; If server login failed (server_connect_timeout or auth failure) +;; then wait this many second. +;server_login_retry = 15 + +;; Dangerous. Server connection is closed if query does not return +;; in this time. Should be used to survive network problems, +;; _not_ as statement_timeout. (default: 0) +;query_timeout = 0 + +;; Dangerous. Client connection is closed if the query is not assigned +;; to a server in this time. Should be used to limit the number of queued +;; queries in case of a database or network failure. (default: 120) +;query_wait_timeout = 120 + +;; Dangerous. Client connection is closed if no activity in this time. +;; Should be used to survive network problems. (default: 0) +;client_idle_timeout = 0 + +;; Disconnect clients who have not managed to log in after connecting +;; in this many seconds. +;client_login_timeout = 60 + +;; Clean automatically created database entries (via "*") if they +;; stay unused in this many seconds. +; autodb_idle_timeout = 3600 + +;; How long SUSPEND/-R waits for buffer flush before closing connection. +;suspend_timeout = 10 + +;; Close connections which are in "IDLE in transaction" state longer than +;; this many seconds. +;idle_transaction_timeout = 0 + +;;; +;;; Low-level tuning options +;;; + +;; buffer for streaming packets +;pkt_buf = 4096 + +;; man 2 listen +;listen_backlog = 128 + +;; Max number pkt_buf to process in one event loop. +;sbuf_loopcnt = 5 + +;; Maximum PostgreSQL protocol packet size. +;max_packet_size = 2147483647 + +;; networking options, for info: man 7 tcp + +;; Linux: notify program about new connection only if there +;; is also data received. (Seconds to wait.) +;; On Linux the default is 45, on other OS'es 0. +;tcp_defer_accept = 0 + +;; In-kernel buffer size (Linux default: 4096) +;tcp_socket_buffer = 0 + +;; whether tcp keepalive should be turned on (0/1) +;tcp_keepalive = 1 + +;; The following options are Linux-specific. +;; They also require tcp_keepalive=1. + +;; count of keepalive packets +;tcp_keepcnt = 0 + +;; how long the connection can be idle, +;; before sending keepalive packets +;tcp_keepidle = 0 + +;; The time between individual keepalive probes. +;tcp_keepintvl = 0 + +;; DNS lookup caching time +;dns_max_ttl = 15 + +;; DNS zone SOA lookup period +;dns_zone_check_period = 0 + +;; DNS negative result caching time +;dns_nxdomain_ttl = 15 + +;;; +;;; Random stuff +;;; + +;; Hackish security feature. Helps against SQL-injection - when PQexec is disabled, +;; multi-statement cannot be made. +;disable_pqexec = 0 + +;; Config file to use for next RELOAD/SIGHUP. +;; By default contains config file from command line. +;conffile + +;; Win32 service name to register as. job_name is alias for service_name, +;; used by some Skytools scripts. +;service_name = pgbouncer +;job_name = pgbouncer + +;; Read additional config from the /etc/pgbouncer/pgbouncer-other.ini file +;%include /etc/pgbouncer/pgbouncer-other.ini diff --git a/roles/newsreader/templates/rabbitmq.conf.j2 b/roles/newsreader/templates/rabbitmq.conf.j2 new file mode 100644 index 0000000..e2017c6 --- /dev/null +++ b/roles/newsreader/templates/rabbitmq.conf.j2 @@ -0,0 +1,18 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +# Defaults to rabbit. This can be useful if you want to run more than one node +# per machine - RABBITMQ_NODENAME should be unique per erlang-node-and-machine +# combination. See the clustering on a single machine guide for details: +# http://www.rabbitmq.com/clustering.html#single-machine +#NODENAME=rabbit + +# By default RabbitMQ will bind to all interfaces, on IPv4 and IPv6 if +# available. Set this if you only want to bind to one network interface or# +# address family. +#NODE_IP_ADDRESS=127.0.0.1 + +# Defaults to 5672. +#NODE_PORT=5672 + +# Fix rabbitmq name resolution +HOSTNAME=localhost diff --git a/roles/newsreader/templates/sudoers.j2 b/roles/newsreader/templates/sudoers.j2 new file mode 100644 index 0000000..5d0a35c --- /dev/null +++ b/roles/newsreader/templates/sudoers.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} {{ ansible_date_time.time }} {{ ansible_date_time.date }} +# +ansible ALL = (newsreader:newsreader) NOPASSWD: ALL diff --git a/roles/requirements.yml b/roles/requirements.yml new file mode 100644 index 0000000..276b569 --- /dev/null +++ b/roles/requirements.yml @@ -0,0 +1,8 @@ +- src: git+https://git.fudiggity.nl/ansible/common.git + name: common + version: master + scm: git +- src: git+https://git.fudiggity.nl/ansible/npm.git + name: npm + version: master + scm: git