Move to simpler file structure

This commit is contained in:
Sonny Bakker 2021-01-28 21:32:10 +01:00
parent fe72523a5a
commit d71a28e1c8
33 changed files with 374 additions and 406 deletions

View file

@ -3,40 +3,24 @@ stages:
- test
cache:
key: "$CI_COMMIT_REF_SLUG"
key: $CI_COMMIT_REF_SLUG
paths:
- .cache/pip
- node_modules/
lint:
stage: lint
image: python:3.7
before_script:
- pip install ansible ansible-lint --quiet
script:
- ansible-lint playbook.yml
only:
refs:
- development
- merge_requests
pretty-lint:
stage: lint
image: node:12
before_script:
- npm install
- npm install prettier --no-save
script:
- npx prettier "**/*.yml" --check
only:
refs:
- development
- merge_requests
- npx prettier '**/*.yml' --check
syntax-test:
stage: test
image: python:3.7
before_script:
- pip install ansible ansible-lint --quiet
- pip install ansible --quiet
- ansible-galaxy install -r roles/requirements.yml
script:
- ansible-playbook playbook.yml --syntax-check

View file

@ -1,9 +0,0 @@
{
"singleQuote": true,
"printWidth": 90,
"tabWidth": 2,
"useTabs": false,
"bracketSpacing": true,
"parser": "yaml"
}

5
.prettierrc.yml Normal file
View file

@ -0,0 +1,5 @@
singleQuote: true
printWidth: 90
tabWidth: 2
useTabs: false
bracketSpacing: true

View file

@ -1,5 +1,17 @@
- hosts: newsreader
become: yes
become_method: sudo
roles:
- newsreader
tasks:
- import_tasks: 'tasks/main.yml'
- import_tasks: 'tasks/setup.yml'
- import_tasks: 'tasks/poetry.yml'
- import_tasks: 'tasks/project.yml'
handlers:
- import_tasks: 'handlers.yml'
vars_files:
- 'vars/app.yml'
- 'vars/main.yml'
- 'vars/network.yml'
- 'vars/postgres.yml'
- 'vars/reddit.yml'
- 'vars/twitter.yml'

View file

@ -1,6 +0,0 @@
hostname: "rss.fudiggity.nl"
host_interface: "en*"
host_ip: "192.168.178.63"
host_subnet: "24"
host_gateway: "192.168.178.1"
host_dns: "192.168.178.1"

View file

@ -1,17 +0,0 @@
dependencies:
- common
- npm
galaxy_info:
author: sonny
description: "Newsreader installation"
license: "license GPLv3"
min_ansible_version: 2.7
issue_tracker_url: "https://git.fudiggity.nl/sonny/ansible-playbooks/-/issues"
platforms:
- name: Debian
versions:
- buster
galaxy_tags:
- development
- web

View file

@ -1,171 +0,0 @@
- include_role:
name: common
tasks_from: "network.yml"
- include_role:
name: common
tasks_from: "host.yml"
- include_role:
name: common
tasks_from: "sudoers.yml"
loop:
- {
src: "../newsreader/templates/sudoers.j2",
dest: "/etc/sudoers.d/30-ansible-extra",
}
- name: install packages
apt:
name:
- memcached
- pgbouncer
- postfix
- python-psycopg2
- python3-psycopg2
- python3
- python3-pip
- python3-venv
- python3-setuptools
- python3-virtualenv
- python-pip
- python-setuptools
- python-virtualenv
- rabbitmq-server
state: present
notify:
- restart postfix
- name: copy firewall templates
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: "0600"
with_items:
- { src: "nftables.j2", dest: "/etc/nftables.conf" }
notify: restart nftables
- name: copy memcached conf
template:
src: "memcached.j2"
dest: "/etc/memcached.conf"
owner: root
group: root
mode: "0644"
notify: restart memcached
- name: add gitlab to known hosts
become_user: "{{ app_user }}"
known_hosts:
name: "{{ gitlab_domain }}"
key: "{{ gitlab_host_key }}"
- name: add gitlab pubkey
authorized_key:
user: ansible
state: present
key: "{{ app_deploy_key }}"
- name: Add newsreader user
user:
name: "{{ app_user }}"
create_home: yes
shell: /bin/bash
- name: create ssh dir
file:
path: "/home/{{ app_user }}/.ssh"
state: directory
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: 0755
- name: create rabbitmq service override dir
file:
path: /etc/systemd/system/rabbitmq-server.service.d/
state: directory
mode: "0644"
- name: copy rabbitmq configurations
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: "{{ item.owner }}"
group: "{{ item.group }}"
mode: "{{ item.mode }}"
loop:
- {
src: "limits.j2",
dest: "/etc/systemd/system/rabbitmq-server.service.d/limits.conf",
mode: "0644",
group: "root",
owner: "root",
}
- {
src: "rabbitmq.conf.j2",
dest: "/etc/rabbitmq/rabbitmq-env.conf",
mode: "0644",
group: "rabbitmq",
owner: "rabbitmq",
}
notify: restart rabbitmq
- include_role:
name: common
tasks_from: "ssl.yml"
- include_role:
name: common
tasks_from: "nginx.yml"
- name: copy nginx config
template:
src: "nginx.j2"
dest: "/etc/nginx/sites-available/newsreader"
owner: root
group: root
mode: "0644"
- name: link nginx config
file:
src: "/etc/nginx/sites-available/newsreader"
dest: "/etc/nginx/sites-enabled/newsreader"
owner: root
group: root
mode: "0777"
state: link
- name: copy nftables config
template:
src: "nftables.j2"
dest: "/etc/nftables.conf"
owner: root
group: root
mode: "0600"
notify: restart nftables
- name: copy pgbouncer config
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: postgres
group: postgres
mode: "{{ item.mode }}"
loop:
- {
src: "pgbouncer.j2",
dest: "/etc/pgbouncer/pgbouncer.ini",
"mode": "0640",
}
- {
src: "pgbouncer-users.j2",
dest: "/etc/pgbouncer/userlist.txt",
"mode": "0640",
}
- name: ensure pgbouncer is restarted
systemd:
name: pgbouncer
state: restarted
enabled: true
- include_tasks: "project.yml"

View file

@ -1,161 +0,0 @@
- name: create sites dir
file:
path: /srv/sites
state: directory
owner: root
group: root
mode: 0755
- name: create project dir
file:
path: "{{ app_dir }}"
state: directory
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: 0755
- name: clone project
become_user: "{{ app_user }}"
git:
repo: "https://git.fudiggity.nl/sonny/newsreader.git"
dest: "{{ app_dir }}"
version: "{{ app_branch }}"
update: true
force: true
- name: install npm packages
become_user: "{{ app_user }}"
command: /usr/bin/npm install
args:
chdir: "{{ app_dir }}"
- name: build static files
become_user: "{{ app_user }}"
command: /usr/bin/npm run build:prod
args:
chdir: "{{ app_dir }}"
- name: run migrations
become_user: "{{ app_user }}"
django_manage:
command: migrate
app_path: "{{ app_dir }}/src/"
virtualenv: "{{ app_dir }}/.venv"
settings: "newsreader.conf.production"
- name: collect static files
become_user: "{{ app_user }}"
django_manage:
command: collectstatic
app_path: "{{ app_dir }}/src/"
virtualenv: "{{ app_dir }}/.venv"
settings: "newsreader.conf.production"
- name: include poetry tasks
include_role:
name: common
tasks_from: "poetry.yml"
vars:
poetry_user: "{{ app_user }}"
poetry_dir: "/home/{{ app_user }}/.poetry"
- name: run poetry tasks
block:
- name: retrieve user $PATH
shell: "echo $PATH"
become_user: "{{ app_user }}"
register: path_stats
- name: set poetry user variables
set_fact:
poetry_user_path: "{{ path_stats.stdout }}"
- name: set default venv python version
become_user: "{{ app_user }}"
command: "poetry env use python3.7"
args:
chdir: "{{ app_dir }}"
environment:
PATH: "/home/{{ app_user }}/.local/bin:{{ poetry_user_path }}"
- name: install project dependencies
become_user: "{{ app_user }}"
command: 'poetry install --extras "sentry"' # noqa 301
args:
chdir: "{{ app_dir }}"
environment:
PATH: "/home/{{ app_user }}/.local/bin:{{ poetry_user_path }}"
- name: setup env file
template:
src: "env.j2"
dest: "{{ app_dir }}/.env"
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: 0600
- name: setup gunicorn service
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: "{{ item.mode }}"
loop:
- {
src: "gunicorn-socket.j2",
dest: "/etc/systemd/system/gunicorn.socket",
"mode": "0644",
}
- {
src: "gunicorn.j2",
dest: "/etc/systemd/system/gunicorn.service",
"mode": "0644",
}
notify:
- restart gunicorn socket
- stop gunicorn service
- name: create conf dir
become_user: "{{ app_user }}"
file:
path: "/home/{{ app_user }}/.config/conf.d"
state: directory
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: 0750
- name: create celery run dir
file:
path: /run/celery
state: directory
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: 0755
- name: copy celery config
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: "{{ item.mode }}"
loop:
- {
src: "celery.j2",
dest: "/etc/systemd/system/celery.service",
"mode": "0644",
}
- {
src: "celerybeat.j2",
dest: "/etc/systemd/system/celerybeat.service",
"mode": "0644",
}
- {
src: "celery.env.j2",
dest: "/home/newsreader/.config/conf.d/celery",
"mode": "0640",
}
notify:
- restart celery
- restart celerybeat

152
tasks/main.yml Normal file
View file

@ -0,0 +1,152 @@
- include_role:
name: common
tasks_from: 'network.yml'
- include_role:
name: common
tasks_from: 'host.yml'
- include_role:
name: common
tasks_from: 'sudoers.yml'
loop:
- { src: '../../templates/sudoers.j2', dest: '/etc/sudoers.d/30-ansible-extra' }
- name: install packages
apt:
name: '{{ packages }}'
state: present
notify:
- restart postfix
- name: copy firewall templates
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
owner: root
group: root
mode: '0600'
loop:
- { src: 'templates/nftables.j2', dest: '/etc/nftables.conf' }
notify: restart nftables
- name: copy memcached conf
template:
src: 'templates/memcached.j2'
dest: '/etc/memcached.conf'
owner: root
group: root
mode: '0644'
notify: restart memcached
- name: add gitlab to known hosts
become_user: '{{ app_user }}'
known_hosts:
name: '{{ gitlab_domain }}'
key: '{{ gitlab_host_key }}'
- name: add gitlab pubkey
authorized_key:
user: ansible
state: present
key: '{{ app_deploy_key }}'
- name: Add newsreader user
user:
name: '{{ app_user }}'
create_home: yes
shell: /bin/bash
- name: create ssh dir
file:
path: '/home/{{ app_user }}/.ssh'
state: directory
owner: '{{ app_user }}'
group: '{{ app_user }}'
mode: 0755
- name: create rabbitmq service override dir
file:
path: /etc/systemd/system/rabbitmq-server.service.d/
state: directory
mode: '0644'
- name: copy rabbitmq configurations
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
owner: '{{ item.owner }}'
group: '{{ item.group }}'
mode: '{{ item.mode }}'
loop:
- {
src: 'templates/limits.j2',
dest: '/etc/systemd/system/rabbitmq-server.service.d/limits.conf',
mode: '0644',
group: 'root',
owner: 'root',
}
- {
src: 'rabbitmq.conf.j2',
dest: '/etc/rabbitmq/rabbitmq-env.conf',
mode: '0644',
group: 'rabbitmq',
owner: 'rabbitmq',
}
notify: restart rabbitmq
- include_role:
name: common
tasks_from: 'ssl.yml'
- include_role:
name: common
tasks_from: 'nginx.yml'
- name: copy nginx config
template:
src: 'templates/nginx.j2'
dest: '/etc/nginx/sites-available/newsreader'
owner: root
group: root
mode: '0644'
- name: link nginx config
file:
src: '/etc/nginx/sites-available/newsreader'
dest: '/etc/nginx/sites-enabled/newsreader'
owner: root
group: root
mode: '0777'
state: link
- name: copy nftables config
template:
src: 'templates/nftables.j2'
dest: '/etc/nftables.conf'
owner: root
group: root
mode: '0600'
notify: restart nftables
- name: copy pgbouncer config
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
owner: postgres
group: postgres
mode: '{{ item.mode }}'
loop:
- {
src: 'templates/pgbouncer.j2',
dest: '/etc/pgbouncer/pgbouncer.ini',
'mode': '0640',
}
- {
src: 'templates/pgbouncer-users.j2',
dest: '/etc/pgbouncer/userlist.txt',
'mode': '0640',
}
- name: ensure pgbouncer is restarted
systemd:
name: pgbouncer
state: restarted
enabled: true

32
tasks/poetry.yml Normal file
View file

@ -0,0 +1,32 @@
- name: include poetry tasks
include_role:
name: common
tasks_from: 'poetry.yml'
vars:
poetry_user: '{{ app_user }}'
poetry_dir: '/home/{{ app_user }}/.poetry'
- name: retrieve user $PATH
shell: 'echo $PATH'
become_user: '{{ app_user }}'
register: path_stats
- name: set poetry user variables
set_fact:
poetry_user_path: '{{ path_stats.stdout }}'
- name: set default venv python version
become_user: '{{ app_user }}'
command: 'poetry env use python3.7'
args:
chdir: '{{ app_dir }}'
environment:
PATH: '/home/{{ app_user }}/.local/bin:{{ poetry_user_path }}'
- name: install project dependencies
become_user: '{{ app_user }}'
command: 'poetry install --extras "sentry"'
args:
chdir: '{{ app_dir }}'
environment:
PATH: '/home/{{ app_user }}/.local/bin:{{ poetry_user_path }}'

101
tasks/project.yml Normal file
View file

@ -0,0 +1,101 @@
- name: install npm packages
become_user: '{{ app_user }}'
command: /usr/bin/npm install
args:
chdir: '{{ app_dir }}'
- name: build static files
become_user: '{{ app_user }}'
command: /usr/bin/npm run build:prod
args:
chdir: '{{ app_dir }}'
- name: run migrations
become_user: '{{ app_user }}'
django_manage:
command: migrate
app_path: '{{ app_dir }}/src/'
virtualenv: '{{ app_dir }}/.venv'
settings: 'newsreader.conf.production'
- name: collect static files
become_user: '{{ app_user }}'
django_manage:
command: collectstatic
app_path: '{{ app_dir }}/src/'
virtualenv: '{{ app_dir }}/.venv'
settings: 'newsreader.conf.production'
- name: setup env file
template:
src: 'templates/env.j2'
dest: '{{ app_dir }}/.env'
owner: '{{ app_user }}'
group: '{{ app_user }}'
mode: 0600
- name: setup gunicorn service
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
owner: root
group: root
mode: '{{ item.mode }}'
loop:
- {
src: 'templates/gunicorn-socket.j2',
dest: '/etc/systemd/system/gunicorn.socket',
'mode': '0644',
}
- {
src: 'templates/gunicorn.j2',
dest: '/etc/systemd/system/gunicorn.service',
'mode': '0644',
}
notify:
- restart gunicorn socket
- stop gunicorn service
- name: create conf dir
become_user: '{{ app_user }}'
file:
path: '/home/{{ app_user }}/.config/conf.d'
state: directory
owner: '{{ app_user }}'
group: '{{ app_user }}'
mode: 0750
- name: create celery run dir
file:
path: /run/celery
state: directory
owner: '{{ app_user }}'
group: '{{ app_user }}'
mode: 0755
- name: copy celery config
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
owner: '{{ app_user }}'
group: '{{ app_user }}'
mode: '{{ item.mode }}'
loop:
- {
src: 'templates/celery.j2',
dest: '/etc/systemd/system/celery.service',
'mode': '0644',
}
- {
src: 'templates/celerybeat.j2',
dest: '/etc/systemd/system/celerybeat.service',
'mode': '0644',
}
- {
src: 'templates/celery.env.j2',
dest: '/home/newsreader/.config/conf.d/celery',
'mode': '0640',
}
notify:
- restart celery
- restart celerybeat

24
tasks/setup.yml Normal file
View file

@ -0,0 +1,24 @@
- name: create sites dir
file:
path: '/srv/sites'
state: directory
owner: root
group: root
mode: 0755
- name: create project dir
file:
path: '{{ app_dir }}'
state: directory
owner: '{{ app_user }}'
group: '{{ app_user }}'
mode: 0755
- name: clone project
become_user: '{{ app_user }}'
git:
repo: 'https://git.fudiggity.nl/sonny/newsreader.git'
dest: '{{ app_dir }}'
version: '{{ app_branch }}'
update: true
force: true

View file

@ -1,10 +1,10 @@
app_name: "newsreader"
app_user: "newsreader"
app_branch: "master"
app_dir: "/srv/sites/newsreader"
app_deploy_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICq4U2MKiufVeM8fVzpfoA/rhiWTMnrJr9usAZAG6kfb Key for deploying to newsreader app"
app_name: 'newsreader'
app_user: 'newsreader'
app_branch: 'master'
app_dir: '/srv/sites/newsreader'
app_deploy_key: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICq4U2MKiufVeM8fVzpfoA/rhiWTMnrJr9usAZAG6kfb Key for deploying to newsreader app'
django_settings_module: "newsreader.conf.production"
django_settings_module: 'newsreader.conf.production'
django_secret_key: !vault |
$ANSIBLE_VAULT;1.1;AES256
37383464313435333061393165373731303161343236666138313566333631303839393163313038
@ -15,4 +15,4 @@ django_secret_key: !vault |
35626539383762383462646632616334633737623035643034643433623237323932373334316639
356533316361653939303165313766633666
admins: ""
admins: ''

View file

@ -1,7 +1,23 @@
default_user: "sonny"
default_user: 'sonny'
gitlab_host_key: "git.fudiggity.nl ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICNmvcxza79T7JZMkifmquwXH/kMUqDnKs9Oob+JrRvn"
gitlab_domain: "git.fudiggity.nl"
packages:
- memcached
- pgbouncer
- postfix
- python-psycopg2
- python3-psycopg2
- python3
- python3-pip
- python3-venv
- python3-setuptools
- python3-virtualenv
- python-pip
- python-setuptools
- python-virtualenv
- rabbitmq-server
gitlab_host_key: 'git.fudiggity.nl ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICNmvcxza79T7JZMkifmquwXH/kMUqDnKs9Oob+JrRvn'
gitlab_domain: 'git.fudiggity.nl'
sentry_dsn: !vault |
$ANSIBLE_VAULT;1.1;AES256

6
vars/network.yml Normal file
View file

@ -0,0 +1,6 @@
hostname: 'rss.fudiggity.nl'
host_interface: 'en*'
host_ip: '192.168.178.63'
host_subnet: '24'
host_gateway: '192.168.178.1'
host_dns: '192.168.178.1'

View file

@ -1,7 +1,7 @@
postgres_host: "192.168.178.165"
postgres_port: "5432"
postgres_db: "newsreader"
postgres_user: "newsreader"
postgres_host: '192.168.178.165'
postgres_port: '5432'
postgres_db: 'newsreader'
postgres_user: 'newsreader'
postgres_password: !vault |
$ANSIBLE_VAULT;1.1;AES256
32613132353864633832306363626432343063616433343830623064326166653231313334636463
@ -11,10 +11,10 @@ postgres_password: !vault |
38623566363265373230386535303332363564393234636561663761353235303132373865353530
6138663238346363383737633133383638383962386236343565
pgbouncer_listen_address: "127.0.0.1"
pgbouncer_port: "6432"
pgbouncer_name: "newsreader"
pgbouncer_user: "newsreader"
pgbouncer_listen_address: '127.0.0.1'
pgbouncer_port: '6432'
pgbouncer_name: 'newsreader'
pgbouncer_user: 'newsreader'
pgbouncer_password: !vault |
$ANSIBLE_VAULT;1.1;AES256
38613333336663643735396637373664363432343633653161633463373536653964656266356564

View file

@ -12,4 +12,4 @@ reddit_client_secret: !vault |
36636464353761383464343634323035666163353561383231623337343732326263353535656165
3738633565396265320a343330623938356631376664326562353437333263386538356438653336
64326363666638306337386266653331633938316639383034376464306238613839
reddit_callback_url: "https://rss.fudiggity.nl/accounts/settings/integrations/reddit/callback/"
reddit_callback_url: 'https://rss.fudiggity.nl/accounts/settings/integrations/reddit/callback/'

View file

@ -14,4 +14,4 @@ twitter_client_secret: !vault |
32613037316134643965353138643236636632623865636632363964666161303330336136626264
63366438343633653566313231633739343036663736333037353465353439346135663733363137
386165313662356630643164396563316562
twitter_redirect_url: "https://rss.fudiggity.nl/accounts/settings/integrations/twitter/callback/"
twitter_redirect_url: 'https://rss.fudiggity.nl/accounts/settings/integrations/twitter/callback/'