diff --git a/docker-compose.yml b/docker-compose.yml index c7dc5ca..8ce24e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" volumes: postgres-data: static-files: @@ -16,7 +16,7 @@ services: rabbitmq: image: rabbitmq:3.7 memcached: - image: memcached:1.5.22 + image: memcached:1.6 ports: - "11211:11211" entrypoint: @@ -31,6 +31,7 @@ services: - DJANGO_SETTINGS_MODULE=newsreader.conf.docker depends_on: - rabbitmq + - memcached volumes: - .:/app django: @@ -41,9 +42,10 @@ services: environment: - DJANGO_SETTINGS_MODULE=newsreader.conf.docker ports: - - '8000:8000' + - "8000:8000" depends_on: - db + - memcached volumes: - .:/app - static-files:/app/src/newsreader/static diff --git a/poetry.lock b/poetry.lock index cab45d1..0bbd4e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,40 +1,40 @@ [[package]] -category = "main" -description = "Low-level AMQP client for Python (fork of amqplib)." name = "amqp" +version = "2.5.2" +description = "Low-level AMQP client for Python (fork of amqplib)." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.5.2" [package.dependencies] vine = ">=1.1.3,<5.0.0a1" [[package]] -category = "dev" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." name = "appdirs" +version = "1.4.3" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = "*" -version = "1.4.3" [[package]] -category = "main" -description = "ASGI specs, helper code, and adapters" name = "asgiref" +version = "3.2.7" +description = "ASGI specs, helper code, and adapters" +category = "main" optional = false python-versions = ">=3.5" -version = "3.2.7" [package.extras] tests = ["pytest (>=4.3.0,<4.4.0)", "pytest-asyncio (>=0.10.0,<0.11.0)"] [[package]] -category = "dev" -description = "Classes Without Boilerplate" name = "attrs" +version = "19.3.0" +description = "Classes Without Boilerplate" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "19.3.0" [package.extras] azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] @@ -43,46 +43,49 @@ docs = ["sphinx", "zope.interface"] tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] [[package]] -category = "dev" -description = "Removes unused imports and unused variables" name = "autoflake" +version = "1.3.1" +description = "Removes unused imports and unused variables" +category = "dev" optional = false python-versions = "*" -version = "1.3.1" [package.dependencies] pyflakes = ">=1.1.0" [[package]] -category = "main" -description = "Screen-scraping library" name = "beautifulsoup4" +version = "4.9.0" +description = "Screen-scraping library" +category = "main" optional = false python-versions = "*" -version = "4.9.0" - -[package.dependencies] -soupsieve = [">1.2", "<2.0"] [package.extras] html5lib = ["html5lib"] lxml = ["lxml"] -[[package]] -category = "main" -description = "Python multiprocessing fork with improvements and bugfixes" -name = "billiard" -optional = false -python-versions = "*" -version = "3.6.3.0" +[package.dependencies] +soupsieve = [">1.2", "<2.0"] + +[[package]] +name = "billiard" +version = "3.6.3.0" +description = "Python multiprocessing fork with improvements and bugfixes" +category = "main" +optional = false +python-versions = "*" [[package]] -category = "dev" -description = "The uncompromising code formatter." name = "black" +version = "19.3b0" +description = "The uncompromising code formatter." +category = "dev" optional = false python-versions = ">=3.6" -version = "19.3b0" + +[package.extras] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [package.dependencies] appdirs = "*" @@ -90,34 +93,25 @@ attrs = ">=18.1.0" click = ">=6.5" toml = ">=0.9.4" -[package.extras] -d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] - [[package]] -category = "main" -description = "An easy safelist-based HTML-sanitizing tool." name = "bleach" +version = "3.1.4" +description = "An easy safelist-based HTML-sanitizing tool." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "3.1.4" [package.dependencies] six = ">=1.9.0" webencodings = "*" [[package]] -category = "main" -description = "Distributed Task Queue." name = "celery" +version = "4.4.2" +description = "Distributed Task Queue." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*," -version = "4.4.2" - -[package.dependencies] -billiard = ">=3.6.3.0,<4.0" -kombu = ">=4.6.8,<4.7" -pytz = ">0.0-dev" -vine = "1.3.0" [package.extras] arangodb = ["pyArango (>=1.3.2)"] @@ -153,37 +147,43 @@ yaml = ["PyYAML (>=3.10)"] zookeeper = ["kazoo (>=1.3.1)"] zstd = ["zstandard"] +[package.dependencies] +billiard = ">=3.6.3.0,<4.0" +kombu = ">=4.6.8,<4.7" +pytz = ">0.0-dev" +vine = "1.3.0" + [[package]] -category = "main" -description = "Python package for providing Mozilla's CA Bundle." name = "certifi" -optional = false -python-versions = "*" version = "2020.4.5.1" - -[[package]] +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "chardet" +version = "3.0.4" +description = "Universal encoding detector for Python 2 and 3" category = "main" -description = "Universal encoding detector for Python 2 and 3" -name = "chardet" optional = false python-versions = "*" -version = "3.0.4" [[package]] -category = "dev" -description = "Composable command line interface toolkit" name = "click" +version = "7.1.1" +description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "7.1.1" [[package]] -category = "main" -description = "Python client library for Core API." name = "coreapi" +version = "2.3.3" +description = "Python client library for Core API." +category = "main" optional = false python-versions = "*" -version = "2.3.3" [package.dependencies] coreschema = "*" @@ -192,62 +192,62 @@ requests = "*" uritemplate = "*" [[package]] -category = "main" -description = "Core Schema." name = "coreschema" +version = "0.0.4" +description = "Core Schema." +category = "main" optional = false python-versions = "*" -version = "0.0.4" [package.dependencies] jinja2 = "*" [[package]] -category = "dev" -description = "Code coverage measurement for Python" name = "coverage" +version = "5.1" +description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "5.1" [package.extras] toml = ["toml"] [[package]] -category = "main" -description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." name = "django" +version = "3.0.7" +description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." +category = "main" optional = false python-versions = ">=3.6" -version = "3.0.7" + +[package.extras] +argon2 = ["argon2-cffi (>=16.1.0)"] +bcrypt = ["bcrypt"] [package.dependencies] asgiref = ">=3.2,<4.0" pytz = "*" sqlparse = ">=0.2.2" -[package.extras] -argon2 = ["argon2-cffi (>=16.1.0)"] -bcrypt = ["bcrypt"] - [[package]] -category = "main" -description = "A helper class for handling configuration defaults of packaged apps gracefully." name = "django-appconf" +version = "1.0.4" +description = "A helper class for handling configuration defaults of packaged apps gracefully." +category = "main" optional = false python-versions = "*" -version = "1.0.4" [package.dependencies] django = "*" [[package]] -category = "main" -description = "Keep track of failed login attempts in Django-powered sites." name = "django-axes" +version = "5.3.1" +description = "Keep track of failed login attempts in Django-powered sites." +category = "main" optional = false python-versions = "~=3.6" -version = "5.3.1" [package.dependencies] django = ">=1.11" @@ -255,93 +255,96 @@ django-appconf = ">=1.0.3" django-ipware = ">=2.0.2" [[package]] -category = "main" -description = "Database-backed Periodic Tasks." name = "django-celery-beat" +version = "2.0.0" +description = "Database-backed Periodic Tasks." +category = "main" optional = false python-versions = "*" -version = "2.0.0" [package.dependencies] -Django = ">=1.11.17" celery = "*" +Django = ">=1.11.17" django-timezone-field = ">=4.0,<5.0" python-crontab = ">=2.3.4" [[package]] -category = "dev" -description = "A configurable set of panels that display various debug information about the current request/response." name = "django-debug-toolbar" +version = "2.2" +description = "A configurable set of panels that display various debug information about the current request/response." +category = "dev" optional = false python-versions = ">=3.5" -version = "2.2" [package.dependencies] Django = ">=1.11" sqlparse = ">=0.2.0" [[package]] -category = "dev" -description = "Extensions for Django" name = "django-extensions" +version = "2.2.9" +description = "Extensions for Django" +category = "dev" optional = false python-versions = "*" -version = "2.2.9" [package.dependencies] six = ">=1.2" [[package]] -category = "main" -description = "A Django utility application that returns client's real IP address" name = "django-ipware" -optional = false -python-versions = "*" version = "2.1.0" - -[[package]] +description = "A Django utility application that returns client's real IP address" category = "main" -description = "An extensible user-registration application for Django" -name = "django-registration-redux" optional = false python-versions = "*" -version = "2.7" [[package]] +name = "django-registration-redux" +version = "2.7" +description = "An extensible user-registration application for Django" category = "main" -description = "A Django app providing database and form fields for pytz timezone objects." +optional = false +python-versions = "*" + +[[package]] name = "django-timezone-field" +version = "4.0" +description = "A Django app providing database and form fields for pytz timezone objects." +category = "main" optional = false python-versions = ">=3.5" -version = "4.0" [package.dependencies] django = ">=2.2" pytz = "*" [[package]] -category = "main" -description = "Web APIs for Django, made easy." name = "djangorestframework" +version = "3.11.0" +description = "Web APIs for Django, made easy." +category = "main" optional = false python-versions = ">=3.5" -version = "3.11.0" [package.dependencies] django = ">=1.11" [[package]] -category = "main" -description = "Automated generation of real Swagger/OpenAPI 2.0 schemas from Django Rest Framework code." name = "drf-yasg" +version = "1.17.1" +description = "Automated generation of real Swagger/OpenAPI 2.0 schemas from Django Rest Framework code." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "1.17.1" + +[package.extras] +validation = ["swagger-spec-validator (>=2.1.0)"] [package.dependencies] -Django = ">=1.11.7" coreapi = ">=2.3.3" coreschema = ">=0.0.4" +Django = ">=1.11.7" djangorestframework = ">=3.8" inflection = ">=0.3.1" packaging = "*" @@ -349,62 +352,67 @@ packaging = "*" six = ">=1.10.0" uritemplate = ">=3.0.0" -[package.extras] -validation = ["swagger-spec-validator (>=2.1.0)"] - [[package]] -category = "dev" -description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." name = "factory-boy" +version = "2.12.0" +description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.12.0" [package.dependencies] Faker = ">=0.7.0" [[package]] -category = "dev" -description = "Faker is a Python package that generates fake data for you." name = "faker" +version = "4.0.2" +description = "Faker is a Python package that generates fake data for you." +category = "dev" optional = false python-versions = ">=3.4" -version = "4.0.2" [package.dependencies] python-dateutil = ">=2.4" text-unidecode = "1.3" [[package]] -category = "main" -description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" name = "feedparser" +version = "5.2.1" +description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" +category = "main" optional = false python-versions = "*" -version = "5.2.1" [[package]] -category = "dev" -description = "Let your Python tests travel through time" name = "freezegun" +version = "0.3.15" +description = "Let your Python tests travel through time" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.3.15" [package.dependencies] python-dateutil = ">=1.0,<2.0 || >2.0" six = "*" [[package]] +name = "ftfy" +version = "5.8" +description = "Fixes some problems with Unicode text after the fact" category = "main" -description = "WSGI HTTP Server for UNIX" -name = "gunicorn" optional = false -python-versions = ">=3.4" -version = "20.0.4" +python-versions = ">=3.5" [package.dependencies] -setuptools = ">=3.0" +wcwidth = "*" + +[[package]] +name = "gunicorn" +version = "20.0.4" +description = "WSGI HTTP Server for UNIX" +category = "main" +optional = false +python-versions = ">=3.4" [package.extras] eventlet = ["eventlet (>=0.9.7)"] @@ -412,45 +420,48 @@ gevent = ["gevent (>=0.13)"] setproctitle = ["setproctitle"] tornado = ["tornado (>=0.2)"] +[package.dependencies] +setuptools = ">=3.0" + [[package]] -category = "main" -description = "Internationalized Domain Names in Applications (IDNA)" name = "idna" +version = "2.9" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.9" [[package]] -category = "main" -description = "Read metadata from Python packages" -marker = "python_version < \"3.8\"" name = "importlib-metadata" +version = "1.6.0" +description = "Read metadata from Python packages" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.6.0" - -[package.dependencies] -zipp = ">=0.5" +marker = "python_version < \"3.8\"" [package.extras] docs = ["sphinx", "rst.linker"] testing = ["packaging", "importlib-resources"] -[[package]] -category = "main" -description = "A port of Ruby on Rails inflector to Python" -name = "inflection" -optional = false -python-versions = ">=3.5" -version = "0.4.0" +[package.dependencies] +zipp = ">=0.5" + +[[package]] +name = "inflection" +version = "0.4.0" +description = "A port of Ruby on Rails inflector to Python" +category = "main" +optional = false +python-versions = ">=3.5" [[package]] -category = "dev" -description = "A Python utility / library to sort Python imports." name = "isort" +version = "4.3.21" +description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "4.3.21" [package.extras] pipfile = ["pipreqs", "requirementslib"] @@ -459,41 +470,34 @@ requirements = ["pipreqs", "pip-api"] xdg_home = ["appdirs (>=1.4.0)"] [[package]] -category = "main" -description = "Simple immutable types for python." name = "itypes" +version = "1.1.0" +description = "Simple immutable types for python." +category = "main" optional = false python-versions = "*" -version = "1.1.0" [[package]] -category = "main" -description = "A very fast and expressive template engine." name = "jinja2" +version = "2.11.1" +description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.11.1" - -[package.dependencies] -MarkupSafe = ">=0.23" [package.extras] i18n = ["Babel (>=0.8)"] +[package.dependencies] +MarkupSafe = ">=0.23" + [[package]] -category = "main" -description = "Messaging library for Python." name = "kombu" +version = "4.6.8" +description = "Messaging library for Python." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "4.6.8" - -[package.dependencies] -amqp = ">=2.5.2,<2.6" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.18" [package.extras] azureservicebus = ["azure-servicebus (>=0.21.1)"] @@ -511,13 +515,20 @@ sqs = ["boto3 (>=1.4.4)", "pycurl (7.43.0.2)"] yaml = ["PyYAML (>=3.10)"] zookeeper = ["kazoo (>=1.3.1)"] +[package.dependencies] +amqp = ">=2.5.2,<2.6" + +[package.dependencies.importlib-metadata] +version = ">=0.18" +python = "<3.8" + [[package]] -category = "main" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." name = "lxml" +version = "4.5.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" -version = "4.5.0" [package.extras] cssselect = ["cssselect (>=0.7)"] @@ -526,112 +537,129 @@ htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] [[package]] -category = "main" -description = "Safely add untrusted strings to HTML/XML markup." name = "markupsafe" +version = "1.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -version = "1.1.1" [[package]] +name = "oauthlib" +version = "3.1.0" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" category = "main" -description = "Core utilities for Python packages" -name = "packaging" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +rsa = ["cryptography"] +signals = ["blinker"] +signedtoken = ["cryptography", "pyjwt (>=1.0.0)"] + +[[package]] +name = "packaging" version = "20.3" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" six = "*" [[package]] -category = "main" -description = "psycopg2 - Python-PostgreSQL Database Adapter" name = "psycopg2-binary" +version = "2.8.5" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -version = "2.8.5" [[package]] -category = "dev" -description = "passive checker of Python programs" name = "pyflakes" +version = "2.2.0" +description = "passive checker of Python programs" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.2.0" [[package]] -category = "main" -description = "Python parsing module" name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "2.4.7" [[package]] -category = "main" -description = "Python Crontab API" name = "python-crontab" +version = "2.4.1" +description = "Python Crontab API" +category = "main" optional = false python-versions = "*" -version = "2.4.1" - -[package.dependencies] -python-dateutil = "*" [package.extras] cron-description = ["cron-descriptor"] cron-schedule = ["croniter"] +[package.dependencies] +python-dateutil = "*" + [[package]] -category = "main" -description = "Extensions to the standard Python datetime module" name = "python-dateutil" +version = "2.8.1" +description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -version = "2.8.1" [package.dependencies] six = ">=1.5" [[package]] -category = "main" -description = "Add .env support to your django/flask apps in development and deployments" name = "python-dotenv" +version = "0.12.0" +description = "Add .env support to your django/flask apps in development and deployments" +category = "main" optional = false python-versions = "*" -version = "0.12.0" [package.extras] cli = ["click (>=5.0)"] [[package]] -category = "main" -description = "Pure python memcached client" name = "python-memcached" +version = "1.59" +description = "Pure python memcached client" +category = "main" optional = false python-versions = "*" -version = "1.59" [package.dependencies] six = ">=1.4.0" [[package]] -category = "main" -description = "World timezone definitions, modern and historical" name = "pytz" +version = "2019.3" +description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" -version = "2019.3" [[package]] -category = "main" -description = "Python HTTP for Humans." name = "requests" +version = "2.23.0" +description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.23.0" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [package.dependencies] certifi = ">=2017.4.17" @@ -639,47 +667,54 @@ chardet = ">=3.0.2,<4" idna = ">=2.5,<3" urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" -[package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] - [[package]] +name = "requests-oauthlib" +version = "1.3.0" +description = "OAuthlib authentication support for Requests." category = "main" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -name = "ruamel.yaml" optional = false -python-versions = "*" -version = "0.16.10" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +rsa = ["oauthlib (>=3.0.0)"] [package.dependencies] -[package.dependencies."ruamel.yaml.clib"] -python = "<3.9" -version = ">=0.1.2" +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[[package]] +name = "ruamel.yaml" +version = "0.16.10" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +category = "main" +optional = false +python-versions = "*" [package.extras] docs = ["ryd"] jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] -[[package]] -category = "main" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -marker = "platform_python_implementation == \"CPython\" and python_version < \"3.9\"" -name = "ruamel.yaml.clib" -optional = false -python-versions = "*" -version = "0.2.0" - -[[package]] -category = "main" -description = "Python client for Sentry (https://getsentry.com)" -name = "sentry-sdk" -optional = false -python-versions = "*" -version = "0.15.1" - [package.dependencies] -certifi = "*" -urllib3 = ">=1.10.0" +[package.dependencies."ruamel.yaml.clib"] +version = ">=0.1.2" +python = "<3.9" + +[[package]] +name = "ruamel.yaml.clib" +version = "0.2.0" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = "*" +marker = "platform_python_implementation == \"CPython\" and python_version < \"3.9\"" + +[[package]] +name = "sentry-sdk" +version = "0.15.1" +description = "Python client for Sentry (https://getsentry.com)" +category = "main" +optional = false +python-versions = "*" [package.extras] aiohttp = ["aiohttp (>=3.5)"] @@ -695,69 +730,73 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] tornado = ["tornado (>=5)"] +[package.dependencies] +certifi = "*" +urllib3 = ">=1.10.0" + [[package]] -category = "main" -description = "Python 2 and 3 compatibility utilities" name = "six" +version = "1.14.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.14.0" [[package]] -category = "main" -description = "A modern CSS selector implementation for Beautiful Soup." name = "soupsieve" +version = "1.9.5" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" optional = false python-versions = "*" -version = "1.9.5" [[package]] -category = "main" -description = "Non-validating SQL parser" name = "sqlparse" +version = "0.3.1" +description = "Non-validating SQL parser" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.3.1" [[package]] -category = "dev" -description = "Traceback serialization library." name = "tblib" +version = "1.6.0" +description = "Traceback serialization library." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "1.6.0" [[package]] -category = "dev" -description = "The most basic Text::Unidecode port" name = "text-unidecode" -optional = false -python-versions = "*" version = "1.3" - -[[package]] +description = "The most basic Text::Unidecode port" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "toml" +version = "0.10.0" +description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" -description = "Python Library for Tom's Obvious, Minimal Language" -name = "toml" optional = false python-versions = "*" -version = "0.10.0" [[package]] -category = "main" -description = "URI templates" name = "uritemplate" +version = "3.0.1" +description = "URI templates" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "3.0.1" [[package]] -category = "main" -description = "HTTP library with thread-safe connection pooling, file post, and more." name = "urllib3" +version = "1.25.8" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "1.25.8" [package.extras] brotli = ["brotlipy (>=0.6.0)"] @@ -765,37 +804,46 @@ secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "cer socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] -category = "main" -description = "Promises, promises, promises." name = "vine" +version = "1.3.0" +description = "Promises, promises, promises." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.3.0" [[package]] +name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" category = "main" -description = "Character encoding aliases for legacy web content" -name = "webencodings" optional = false python-versions = "*" -version = "0.5.1" [[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" category = "main" -description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" +optional = false +python-versions = "*" + +[[package]] name = "zipp" +version = "3.1.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.6" -version = "3.1.0" +marker = "python_version < \"3.8\"" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "6b207d452b10de2399c4c49118da997dda6ed1bb0437963c3f415ecd3d806fe5" +lock-version = "1.0" python-versions = "^3.7" +content-hash = "cda651cbf92ffc53c6ef09bea6204f5927b5a1bf3feff85bc70fa672e526cc91" [metadata.files] amqp = [ @@ -951,6 +999,9 @@ freezegun = [ {file = "freezegun-0.3.15-py2.py3-none-any.whl", hash = "sha256:82c757a05b7c7ca3e176bfebd7d6779fd9139c7cb4ef969c38a28d74deef89b2"}, {file = "freezegun-0.3.15.tar.gz", hash = "sha256:e2062f2c7f95cc276a834c22f1a17179467176b624cc6f936e8bc3be5535ad1b"}, ] +ftfy = [ + {file = "ftfy-5.8.tar.gz", hash = "sha256:51c7767f8c4b47d291fcef30b9625fb5341c06a31e6a3b627039c706c42f3720"}, +] gunicorn = [ {file = "gunicorn-20.0.4-py2.py3-none-any.whl", hash = "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"}, {file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"}, @@ -1046,6 +1097,10 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] +oauthlib = [ + {file = "oauthlib-3.1.0-py2.py3-none-any.whl", hash = "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea"}, + {file = "oauthlib-3.1.0.tar.gz", hash = "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889"}, +] packaging = [ {file = "packaging-20.3-py2.py3-none-any.whl", hash = "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752"}, {file = "packaging-20.3.tar.gz", hash = "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3"}, @@ -1110,9 +1165,15 @@ pytz = [ {file = "pytz-2019.3.tar.gz", hash = "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"}, ] requests = [ + {file = "requests-2.23.0-py2.7.egg", hash = "sha256:5d2d0ffbb515f39417009a46c14256291061ac01ba8f875b90cad137de83beb4"}, {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, ] +requests-oauthlib = [ + {file = "requests-oauthlib-1.3.0.tar.gz", hash = "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a"}, + {file = "requests_oauthlib-1.3.0-py2.py3-none-any.whl", hash = "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d"}, + {file = "requests_oauthlib-1.3.0-py3.7.egg", hash = "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"}, +] "ruamel.yaml" = [ {file = "ruamel.yaml-0.16.10-py2.py3-none-any.whl", hash = "sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b"}, {file = "ruamel.yaml-0.16.10.tar.gz", hash = "sha256:099c644a778bf72ffa00524f78dd0b6476bca94a1da344130f4bf3381ce5b954"}, @@ -1179,6 +1240,10 @@ vine = [ {file = "vine-1.3.0-py2.py3-none-any.whl", hash = "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"}, {file = "vine-1.3.0.tar.gz", hash = "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87"}, ] +wcwidth = [ + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, +] webencodings = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, diff --git a/pyproject.toml b/pyproject.toml index bdc34a9..2d400ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,8 @@ gunicorn = "^20.0.4" python-dotenv = "^0.12.0" django = ">=3.0.7" sentry-sdk = "^0.15.1" +ftfy = "^5.8" +requests_oauthlib = "^1.3.0" [tool.poetry.dev-dependencies] factory-boy = "^2.12.0" diff --git a/src/newsreader/accounts/admin.py b/src/newsreader/accounts/admin.py index 49390c7..02d372c 100644 --- a/src/newsreader/accounts/admin.py +++ b/src/newsreader/accounts/admin.py @@ -11,8 +11,18 @@ class UserAdminForm(UserChangeForm): class Meta: widgets = { "email": forms.EmailInput(attrs={"size": "50"}), - "reddit_access_token": forms.TextInput(attrs={"size": "90"}), - "reddit_refresh_token": forms.TextInput(attrs={"size": "90"}), + "reddit_access_token": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), + "reddit_refresh_token": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), + "twitter_oauth_token": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), + "twitter_oauth_token_secret": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), } @@ -34,6 +44,10 @@ class UserAdmin(DjangoUserAdmin): _("Reddit settings"), {"fields": ("reddit_access_token", "reddit_refresh_token")}, ), + ( + _("Twitter settings"), + {"fields": ("twitter_oauth_token", "twitter_oauth_token_secret")}, + ), ( _("Permission settings"), {"classes": ("collapse",), "fields": ("is_staff", "is_superuser")}, diff --git a/src/newsreader/accounts/migrations/0011_auto_20200913_2101.py b/src/newsreader/accounts/migrations/0011_auto_20200913_2101.py new file mode 100644 index 0000000..b6a83dd --- /dev/null +++ b/src/newsreader/accounts/migrations/0011_auto_20200913_2101.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.7 on 2020-09-13 19:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("accounts", "0010_auto_20200603_2230")] + + operations = [ + migrations.AddField( + model_name="user", + name="twitter_oauth_token", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="user", + name="twitter_oauth_token_secret", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/src/newsreader/accounts/migrations/0012_remove_user_task.py b/src/newsreader/accounts/migrations/0012_remove_user_task.py new file mode 100644 index 0000000..250d300 --- /dev/null +++ b/src/newsreader/accounts/migrations/0012_remove_user_task.py @@ -0,0 +1,10 @@ +# Generated by Django 3.0.7 on 2020-09-26 15:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("accounts", "0011_auto_20200913_2101")] + + operations = [migrations.RemoveField(model_name="user", name="task")] diff --git a/src/newsreader/accounts/models.py b/src/newsreader/accounts/models.py index b8aaa64..2451445 100644 --- a/src/newsreader/accounts/models.py +++ b/src/newsreader/accounts/models.py @@ -1,11 +1,9 @@ -import json - from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import UserManager as DjangoUserManager from django.db import models from django.utils.translation import gettext as _ -from django_celery_beat.models import IntervalSchedule, PeriodicTask +from django_celery_beat.models import PeriodicTask class UserManager(DjangoUserManager): @@ -41,18 +39,12 @@ class UserManager(DjangoUserManager): class User(AbstractUser): email = models.EmailField(_("email address"), unique=True) - task = models.OneToOneField( - PeriodicTask, - on_delete=models.CASCADE, - null=True, - blank=True, - editable=False, - verbose_name="collection task", - ) - reddit_refresh_token = models.CharField(max_length=255, blank=True, null=True) reddit_access_token = models.CharField(max_length=255, blank=True, null=True) + twitter_oauth_token = models.CharField(max_length=255, blank=True, null=True) + twitter_oauth_token_secret = models.CharField(max_length=255, blank=True, null=True) + username = None objects = UserManager() @@ -60,24 +52,12 @@ class User(AbstractUser): USERNAME_FIELD = "email" REQUIRED_FIELDS = [] - def save(self, *args, **kwargs): - super().save(*args, **kwargs) - - if not self.task: - task_interval, _ = IntervalSchedule.objects.get_or_create( - every=1, period=IntervalSchedule.HOURS - ) - - self.task, _ = PeriodicTask.objects.get_or_create( - enabled=True, - interval=task_interval, - name=f"{self.email}-collection-task", - task="FeedTask", - args=json.dumps([self.pk]), - ) - - self.save() - def delete(self, *args, **kwargs): - self.task.delete() + tasks = PeriodicTask.objects.filter(name__contains=self.email) + tasks.delete() + return super().delete(*args, **kwargs) + + @property + def has_twitter_auth(self): + return self.twitter_oauth_token and self.twitter_oauth_token_secret diff --git a/src/newsreader/accounts/templates/accounts/components/settings-form.html b/src/newsreader/accounts/templates/accounts/components/settings-form.html index 7942354..51d4450 100644 --- a/src/newsreader/accounts/templates/accounts/components/settings-form.html +++ b/src/newsreader/accounts/templates/accounts/components/settings-form.html @@ -3,28 +3,15 @@ {% block actions %}
-
- {% include "components/form/cancel-button.html" %} -
-
{% trans "Change password" %} + + {% trans "Third party integrations" %} + {% include "components/form/confirm-button.html" %} - - {% if reddit_authorization_url %} - - {% trans "Authorize Reddit account" %} - - {% endif %} - - {% if reddit_refresh_url %} - - {% trans "Refresh Reddit access token" %} - - {% endif %}
{% endblock actions %} diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html new file mode 100644 index 0000000..4429f02 --- /dev/null +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -0,0 +1,70 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block content %} +
+
+ {% include "components/header/header.html" with title="Integrations" only %} + +
+

Reddit

+
+ {% if reddit_authorization_url %} + + {% trans "Authorize account" %} + + {% else %} + + {% endif %} + + {% if reddit_refresh_url %} + + {% trans "Refresh token" %} + + {% else %} + + {% endif %} + + {% if reddit_revoke_url %} + + {% trans "Deauthorize account" %} + + {% else %} + + {% endif %} +
+
+ +
+

Twitter

+
+ {% if twitter_auth_url %} + + {% else %} + + {% endif %} + + {% if twitter_revoke_url %} + + {% else %} + + {% endif %} +
+
+
+
+{% endblock %} diff --git a/src/newsreader/accounts/templates/accounts/views/reddit.html b/src/newsreader/accounts/templates/accounts/views/reddit.html index b393bbe..5d4f539 100644 --- a/src/newsreader/accounts/templates/accounts/views/reddit.html +++ b/src/newsreader/accounts/templates/accounts/views/reddit.html @@ -1,17 +1,20 @@ {% extends "base.html" %} +{% load i18n %} {% block content %} -
+
{% if error %} -

Reddit authorization failed

+

{% trans "Reddit authorization failed" %}

{{ error }}

{% elif access_token and refresh_token %} -

Reddit account is linked

-

Your reddit account was successfully linked.

+

{% trans "Reddit account is linked" %}

+

{% trans "Your reddit account was successfully linked." %}

{% endif %} -

Return to settings page

+

+ {% trans "Return to integrations page" %} +

{% endblock %} diff --git a/src/newsreader/accounts/templates/accounts/views/twitter.html b/src/newsreader/accounts/templates/accounts/views/twitter.html new file mode 100644 index 0000000..e2c51aa --- /dev/null +++ b/src/newsreader/accounts/templates/accounts/views/twitter.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block content %} +
+
+ {% if error %} +

{% trans "Twitter authorization failed" %}

+

{{ error }}

+ {% elif authorized %} +

{% trans "Twitter account is linked" %}

+

{% trans "Your Twitter account was successfully linked." %}

+ {% endif %} + +

+ {% trans "Return to integrations page" %} +

+
+
+{% endblock %} diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py new file mode 100644 index 0000000..cdc9546 --- /dev/null +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -0,0 +1,537 @@ +from unittest.mock import Mock, patch +from urllib.parse import urlencode +from uuid import uuid4 + +from django.core.cache import cache +from django.test import TestCase +from django.urls import reverse +from django.utils.translation import gettext as _ + +from bs4 import BeautifulSoup + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.news.collection.exceptions import ( + StreamException, + StreamTooManyException, +) +from newsreader.news.collection.twitter import TWITTER_AUTH_URL + + +class IntegrationsViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.url = reverse("accounts:integrations") + + +class RedditIntegrationsTestCase(IntegrationsViewTestCase): + def test_reddit_authorization(self): + self.user.reddit_refresh_token = None + self.user.save() + + response = self.client.get(self.url) + + soup = BeautifulSoup(response.content, features="lxml") + button = soup.find("a", class_="link button button--reddit") + + self.assertEquals(button.text.strip(), "Authorize account") + + def test_reddit_refresh_token(self): + self.user.reddit_refresh_token = "jadajadajada" + self.user.reddit_access_token = None + self.user.save() + + response = self.client.get(self.url) + + soup = BeautifulSoup(response.content, features="lxml") + button = soup.find("a", class_="link button button--reddit") + + self.assertEquals(button.text.strip(), "Refresh token") + + def test_reddit_revoke(self): + self.user.reddit_refresh_token = "jadajadajada" + self.user.reddit_access_token = None + self.user.save() + + response = self.client.get(self.url) + + soup = BeautifulSoup(response.content, features="lxml") + buttons = soup.find_all("a", class_="link button button--reddit") + + self.assertIn( + "Deauthorize account", [button.text.strip() for button in buttons] + ) + + +class RedditTemplateViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.base_url = reverse("accounts:reddit-template") + self.state = str(uuid4()) + + self.patch = patch("newsreader.news.collection.reddit.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + patch.stopall() + + def test_simple(self): + response = self.client.get(self.base_url) + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Return to integrations page") + + def test_successful_authorization(self): + self.mocked_post.return_value.json.return_value = { + "access_token": "1001010412", + "refresh_token": "134510143", + } + + cache.set(f"{self.user.email}-reddit-auth", self.state) + + params = {"state": self.state, "code": "Valid code"} + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.mocked_post.assert_called_once() + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Your reddit account was successfully linked.") + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, "1001010412") + self.assertEquals(self.user.reddit_refresh_token, "134510143") + + self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), None) + + def test_error(self): + params = {"error": "Denied authorization"} + + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Denied authorization") + + def test_invalid_state(self): + cache.set(f"{self.user.email}-reddit-auth", str(uuid4())) + + params = {"code": "Valid code", "state": "Invalid state"} + + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.assertEquals(response.status_code, 200) + self.assertContains( + response, "The saved state for Reddit authorization did not match" + ) + + def test_stream_error(self): + self.mocked_post.side_effect = StreamTooManyException + + cache.set(f"{self.user.email}-reddit-auth", self.state) + + params = {"state": self.state, "code": "Valid code"} + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.mocked_post.assert_called_once() + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Too many requests") + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, None) + self.assertEquals(self.user.reddit_refresh_token, None) + + self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) + + def test_unexpected_json(self): + self.mocked_post.return_value.json.return_value = {"message": "Happy eastern"} + + cache.set(f"{self.user.email}-reddit-auth", self.state) + + params = {"state": self.state, "code": "Valid code"} + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.mocked_post.assert_called_once() + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Access and refresh token not found in response") + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, None) + self.assertEquals(self.user.reddit_refresh_token, None) + + self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) + + +class RedditTokenRedirectViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.RedditTokenTask") + self.mocked_task = self.patch.start() + + def tearDown(self): + cache.clear() + + def test_simple(self): + response = self.client.get(reverse("accounts:reddit-refresh")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_task.delay.assert_called_once_with(self.user.pk) + + self.assertEquals(1, cache.get(f"{self.user.email}-reddit-refresh")) + + def test_not_active(self): + cache.set(f"{self.user.email}-reddit-refresh", 1) + + response = self.client.get(reverse("accounts:reddit-refresh")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_task.delay.assert_not_called() + + +class RedditRevokeRedirectViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.revoke_reddit_token") + self.mocked_revoke = self.patch.start() + + def test_simple(self): + self.user.reddit_access_token = "jadajadajada" + self.user.reddit_refresh_token = "jadajadajada" + self.user.save() + + self.mocked_revoke.return_value = True + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_revoke.assert_called_once_with(self.user) + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, None) + self.assertEquals(self.user.reddit_refresh_token, None) + + def test_no_refresh_token(self): + self.user.reddit_refresh_token = None + self.user.save() + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_revoke.assert_not_called() + + def test_unsuccessful_response(self): + self.user.reddit_access_token = "jadajadajada" + self.user.reddit_refresh_token = "jadajadajada" + self.user.save() + + self.mocked_revoke.return_value = False + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, "jadajadajada") + self.assertEquals(self.user.reddit_refresh_token, "jadajadajada") + + def test_stream_exception(self): + self.user.reddit_access_token = "jadajadajada" + self.user.reddit_refresh_token = "jadajadajada" + self.user.save() + + self.mocked_revoke.side_effect = StreamException + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, "jadajadajada") + self.assertEquals(self.user.reddit_refresh_token, "jadajadajada") + + +class TwitterRevokeRedirectView(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + patch.stopall() + + def test_simple(self): + self.user.twitter_oauth_token = "jadajadajada" + self.user.twitter_oauth_token_secret = "jadajadajada" + self.user.save() + + response = self.client.get(reverse("accounts:twitter-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + def test_no_authorized_account(self): + self.user.twitter_oauth_token = None + self.user.twitter_oauth_token_secret = None + self.user.save() + + response = self.client.get(reverse("accounts:twitter-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_post.assert_not_called() + + def test_stream_exception(self): + self.user.twitter_oauth_token = "jadajadajada" + self.user.twitter_oauth_token_secret = "jadajadajada" + self.user.save() + + self.mocked_post.side_effect = StreamException + + response = self.client.get(reverse("accounts:twitter-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.twitter_oauth_token, "jadajadajada") + self.assertEquals(self.user.twitter_oauth_token_secret, "jadajadajada") + + +class TwitterAuthRedirectViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + cache.clear() + + def test_simple(self): + self.mocked_post.return_value = Mock( + text="oauth_token=foo&oauth_token_secret=bar" + ) + + response = self.client.get(reverse("accounts:twitter-auth")) + + self.assertRedirects( + response, + f"{TWITTER_AUTH_URL}/?oauth_token=foo", + fetch_redirect_response=False, + ) + + cached_token = cache.get(f"twitter-{self.user.email}-token") + cached_secret = cache.get(f"twitter-{self.user.email}-secret") + + self.assertEquals(cached_token, "foo") + self.assertEquals(cached_secret, "bar") + + def test_stream_exception(self): + self.mocked_post.side_effect = StreamException + + response = self.client.get(reverse("accounts:twitter-auth")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + cached_token = cache.get(f"twitter-{self.user.email}-token") + cached_secret = cache.get(f"twitter-{self.user.email}-secret") + + self.assertIsNone(cached_token) + self.assertIsNone(cached_secret) + + def test_unexpected_contents(self): + self.mocked_post.return_value = Mock(text="foo=bar&oauth_token_secret=bar") + + response = self.client.get(reverse("accounts:twitter-auth")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + cached_token = cache.get(f"twitter-{self.user.email}-token") + cached_secret = cache.get(f"twitter-{self.user.email}-secret") + + self.assertIsNone(cached_token) + self.assertIsNone(cached_secret) + + +class TwitterTemplateViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + cache.clear() + + def test_simple(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + self.mocked_post.return_value = Mock( + text="oauth_token=realtoken&oauth_token_secret=realsecret" + ) + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("Twitter account is linked")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.twitter_oauth_token, "realtoken") + self.assertEquals(self.user.twitter_oauth_token_secret, "realsecret") + + self.assertIsNone(cache.get(f"twitter-{self.user.email}-token")) + self.assertIsNone(cache.get(f"twitter-{self.user.email}-secret")) + + def test_denied(self): + params = {"denied": "true", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("Twitter authorization failed")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.mocked_post.assert_not_called() + + def test_mismatched_token(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "boo", "oauth_verifier": "barfoo"} + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("OAuth tokens failed to match")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.mocked_post.assert_not_called() + + def test_missing_secret(self): + cache.set_many({f"twitter-{self.user.email}-token": "foo"}) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("No matching tokens found for this user")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.mocked_post.assert_not_called() + + def test_stream_exception(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + self.mocked_post.side_effect = StreamException + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("Failed requesting access token")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-token")) + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-secret")) + + def test_unexpected_contents(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + self.mocked_post.return_value = Mock( + text="foobar=boo&oauth_token_secret=realsecret" + ) + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("No credentials found in Twitter response")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-token")) + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-secret")) diff --git a/src/newsreader/accounts/tests/test_settings.py b/src/newsreader/accounts/tests/test_settings.py index d093ea4..42db736 100644 --- a/src/newsreader/accounts/tests/test_settings.py +++ b/src/newsreader/accounts/tests/test_settings.py @@ -1,14 +1,8 @@ -from unittest.mock import patch -from urllib.parse import urlencode -from uuid import uuid4 - -from django.core.cache import cache from django.test import TestCase from django.urls import reverse from newsreader.accounts.models import User from newsreader.accounts.tests.factories import UserFactory -from newsreader.news.collection.exceptions import StreamTooManyException class SettingsViewTestCase(TestCase): @@ -22,7 +16,6 @@ class SettingsViewTestCase(TestCase): response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertContains(response, "Authorize Reddit account") def test_user_credential_change(self): response = self.client.post( @@ -36,126 +29,3 @@ class SettingsViewTestCase(TestCase): self.assertEquals(user.first_name, "First name") self.assertEquals(user.last_name, "Last name") - - def test_linked_reddit_account(self): - self.user.reddit_refresh_token = "test" - self.user.save() - - response = self.client.get(self.url) - - self.assertEquals(response.status_code, 200) - self.assertNotContains(response, "Authorize Reddit account") - - -class RedditTemplateViewTestCase(TestCase): - def setUp(self): - self.user = UserFactory(email="test@test.nl", password="test") - self.client.force_login(self.user) - - self.base_url = reverse("accounts:reddit-template") - self.state = str(uuid4()) - - self.patch = patch("newsreader.news.collection.reddit.post") - self.mocked_post = self.patch.start() - - def tearDown(self): - patch.stopall() - - def test_simple(self): - response = self.client.get(self.base_url) - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Return to settings page") - - def test_successful_authorization(self): - self.mocked_post.return_value.json.return_value = { - "access_token": "1001010412", - "refresh_token": "134510143", - } - - cache.set(f"{self.user.email}-reddit-auth", self.state) - - params = {"state": self.state, "code": "Valid code"} - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.mocked_post.assert_called_once() - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Your reddit account was successfully linked.") - - self.user.refresh_from_db() - - self.assertEquals(self.user.reddit_access_token, "1001010412") - self.assertEquals(self.user.reddit_refresh_token, "134510143") - - self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), None) - - def test_error(self): - params = {"error": "Denied authorization"} - - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Denied authorization") - - def test_invalid_state(self): - cache.set(f"{self.user.email}-reddit-auth", str(uuid4())) - - params = {"code": "Valid code", "state": "Invalid state"} - - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.assertEquals(response.status_code, 200) - self.assertContains( - response, "The saved state for Reddit authorization did not match" - ) - - def test_stream_error(self): - self.mocked_post.side_effect = StreamTooManyException - - cache.set(f"{self.user.email}-reddit-auth", self.state) - - params = {"state": self.state, "code": "Valid code"} - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.mocked_post.assert_called_once() - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Too many requests") - - self.user.refresh_from_db() - - self.assertEquals(self.user.reddit_access_token, None) - self.assertEquals(self.user.reddit_refresh_token, None) - - self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) - - def test_unexpected_json(self): - self.mocked_post.return_value.json.return_value = {"message": "Happy eastern"} - - cache.set(f"{self.user.email}-reddit-auth", self.state) - - params = {"state": self.state, "code": "Valid code"} - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.mocked_post.assert_called_once() - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Access and refresh token not found in response") - - self.user.refresh_from_db() - - self.assertEquals(self.user.reddit_access_token, None) - self.assertEquals(self.user.reddit_refresh_token, None) - - self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) diff --git a/src/newsreader/accounts/tests/tests.py b/src/newsreader/accounts/tests/tests.py index e28dbd3..9f6a20f 100644 --- a/src/newsreader/accounts/tests/tests.py +++ b/src/newsreader/accounts/tests/tests.py @@ -1,22 +1,24 @@ from django.test import TestCase -from django_celery_beat.models import PeriodicTask +from django_celery_beat.models import IntervalSchedule, PeriodicTask -from newsreader.accounts.models import User +from newsreader.accounts.tests.factories import UserFactory class UserTestCase(TestCase): - def test_task_is_created(self): - user = User.objects.create(email="durp@burp.nl", task=None) - task = PeriodicTask.objects.get(name=f"{user.email}-collection-task") - - user.refresh_from_db() - - self.assertEquals(task, user.task) - self.assertEquals(PeriodicTask.objects.count(), 1) - def test_task_is_deleted(self): - user = User.objects.create(email="durp@burp.nl", task=None) + user = UserFactory(email="durp@burp.nl") + + interval = IntervalSchedule.objects.create( + every=1, period=IntervalSchedule.HOURS + ) + PeriodicTask.objects.create( + name=f"{user.email}-feed", task="FeedTask", interval=interval + ) + PeriodicTask.objects.create( + name=f"{user.email}-timeline", task="TwitterTimelineTask", interval=interval + ) + user.delete() self.assertEquals(PeriodicTask.objects.count(), 0) diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index 672cf6d..3cdd1b1 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -5,6 +5,7 @@ from newsreader.accounts.views import ( ActivationCompleteView, ActivationResendView, ActivationView, + IntegrationsView, LoginView, LogoutView, PasswordChangeView, @@ -12,18 +13,24 @@ from newsreader.accounts.views import ( PasswordResetConfirmView, PasswordResetDoneView, PasswordResetView, + RedditRevokeRedirectView, RedditTemplateView, RedditTokenRedirectView, RegistrationClosedView, RegistrationCompleteView, RegistrationView, SettingsView, + TwitterAuthRedirectView, + TwitterRevokeRedirectView, + TwitterTemplateView, ) urlpatterns = [ + # Auth path("login/", LoginView.as_view(), name="login"), path("logout/", LogoutView.as_view(), name="logout"), + # Register path("register/", RegistrationView.as_view(), name="register"), path( "register/complete/", @@ -41,6 +48,7 @@ urlpatterns = [ ActivationView.as_view(), name="activate", ), + # Password path("password-reset/", PasswordResetView.as_view(), name="password-reset"), path( "password-reset/done/", @@ -62,15 +70,42 @@ urlpatterns = [ login_required(PasswordChangeView.as_view()), name="password-change", ), - path("settings/", login_required(SettingsView.as_view()), name="settings"), + # Integrations path( - "settings/reddit/callback/", + "settings/integrations/reddit/callback/", login_required(RedditTemplateView.as_view()), name="reddit-template", ), path( - "settings/reddit/refresh/", + "settings/integrations/reddit/refresh/", login_required(RedditTokenRedirectView.as_view()), name="reddit-refresh", ), + path( + "settings/integrations/reddit/revoke/", + login_required(RedditRevokeRedirectView.as_view()), + name="reddit-revoke", + ), + path( + "settings/integrations/twitter/auth/", + login_required(TwitterAuthRedirectView.as_view()), + name="twitter-auth", + ), + path( + "settings/integrations/twitter/callback/", + login_required(TwitterTemplateView.as_view()), + name="twitter-template", + ), + path( + "settings/integrations/twitter/revoke/", + login_required(TwitterRevokeRedirectView.as_view()), + name="twitter-revoke", + ), + path( + "settings/integrations", + login_required(IntegrationsView.as_view()), + name="integrations", + ), + # Settings + path("settings/", login_required(SettingsView.as_view()), name="settings"), ] diff --git a/src/newsreader/accounts/views.py b/src/newsreader/accounts/views.py deleted file mode 100644 index 4f982a9..0000000 --- a/src/newsreader/accounts/views.py +++ /dev/null @@ -1,210 +0,0 @@ -from django.contrib import messages -from django.contrib.auth import views as django_views -from django.core.cache import cache -from django.shortcuts import render -from django.urls import reverse_lazy -from django.utils.translation import gettext as _ -from django.views.generic import RedirectView, TemplateView -from django.views.generic.edit import FormView, ModelFormMixin - -from registration.backends.default import views as registration_views - -from newsreader.accounts.forms import UserSettingsForm -from newsreader.accounts.models import User -from newsreader.news.collection.exceptions import StreamException -from newsreader.news.collection.reddit import ( - get_reddit_access_token, - get_reddit_authorization_url, -) -from newsreader.news.collection.tasks import RedditTokenTask - - -class LoginView(django_views.LoginView): - template_name = "accounts/views/login.html" - success_url = reverse_lazy("index") - - -class LogoutView(django_views.LogoutView): - next_page = reverse_lazy("accounts:login") - - -# RegistrationView shows a registration form and sends the email -# RegistrationCompleteView shows after filling in the registration form -# ActivationView is send within the activation email and activates the account -# ActivationCompleteView shows the success screen when activation was succesful -# ActivationResendView can be used when activation links are expired -# RegistrationClosedView shows when registration is disabled -class RegistrationView(registration_views.RegistrationView): - disallowed_url = reverse_lazy("accounts:register-closed") - template_name = "registration/registration_form.html" - success_url = reverse_lazy("accounts:register-complete") - - -class RegistrationCompleteView(TemplateView): - template_name = "registration/registration_complete.html" - - -class RegistrationClosedView(TemplateView): - template_name = "registration/registration_closed.html" - - -# Redirects or renders failed activation template -class ActivationView(registration_views.ActivationView): - template_name = "registration/activation_failure.html" - - def get_success_url(self, user): - return ("accounts:activate-complete", (), {}) - - -class ActivationCompleteView(TemplateView): - template_name = "registration/activation_complete.html" - - -# Renders activation form resend or resend_activation_complete -class ActivationResendView(registration_views.ResendActivationView): - template_name = "registration/activation_resend_form.html" - - def render_form_submitted_template(self, form): - """ - Renders resend activation complete template with the submitted email. - - """ - email = form.cleaned_data["email"] - context = {"email": email} - - return render( - self.request, "registration/activation_resend_complete.html", context - ) - - -# PasswordResetView sends the mail -# PasswordResetDoneView shows a success message for the above -# PasswordResetConfirmView checks the link the user clicked and -# prompts for a new password -# PasswordResetCompleteView shows a success message for the above -class PasswordResetView(django_views.PasswordResetView): - template_name = "password-reset/password-reset.html" - subject_template_name = "password-reset/password-reset-subject.txt" - email_template_name = "password-reset/password-reset-email.html" - success_url = reverse_lazy("accounts:password-reset-done") - - -class PasswordResetDoneView(django_views.PasswordResetDoneView): - template_name = "password-reset/password-reset-done.html" - - -class PasswordResetConfirmView(django_views.PasswordResetConfirmView): - template_name = "password-reset/password-reset-confirm.html" - success_url = reverse_lazy("accounts:password-reset-complete") - - -class PasswordResetCompleteView(django_views.PasswordResetCompleteView): - template_name = "password-reset/password-reset-complete.html" - - -class PasswordChangeView(django_views.PasswordChangeView): - template_name = "accounts/views/password-change.html" - success_url = reverse_lazy("accounts:settings") - - -class SettingsView(ModelFormMixin, FormView): - template_name = "accounts/views/settings.html" - success_url = reverse_lazy("accounts:settings") - form_class = UserSettingsForm - model = User - - def get(self, request, *args, **kwargs): - self.object = self.get_object() - return super().get(request, *args, **kwargs) - - def get_object(self, **kwargs): - return self.request.user - - def get_context_data(self, **kwargs): - user = self.request.user - - reddit_authorization_url = None - reddit_refresh_url = None - reddit_task_active = cache.get(f"{user.email}-reddit-refresh") - - if ( - user.reddit_refresh_token - and not user.reddit_access_token - and not reddit_task_active - ): - reddit_refresh_url = reverse_lazy("accounts:reddit-refresh") - - if not user.reddit_refresh_token: - reddit_authorization_url = get_reddit_authorization_url(user) - - return { - **super().get_context_data(**kwargs), - "reddit_authorization_url": reddit_authorization_url, - "reddit_refresh_url": reddit_refresh_url, - } - - def get_form_kwargs(self): - return {**super().get_form_kwargs(), "instance": self.request.user} - - -class RedditTemplateView(TemplateView): - template_name = "accounts/views/reddit.html" - - def get(self, request, *args, **kwargs): - context = self.get_context_data(**kwargs) - - error = request.GET.get("error", None) - state = request.GET.get("state", None) - code = request.GET.get("code", None) - - if error: - return self.render_to_response({**context, "error": error}) - - if not code or not state: - return self.render_to_response(context) - - cached_state = cache.get(f"{request.user.email}-reddit-auth") - - if state != cached_state: - return self.render_to_response( - { - **context, - "error": "The saved state for Reddit authorization did not match", - } - ) - - try: - access_token, refresh_token = get_reddit_access_token(code, request.user) - - return self.render_to_response( - { - **context, - "access_token": access_token, - "refresh_token": refresh_token, - } - ) - except StreamException as e: - return self.render_to_response({**context, "error": str(e)}) - except KeyError: - return self.render_to_response( - {**context, "error": "Access and refresh token not found in response"} - ) - - -class RedditTokenRedirectView(RedirectView): - url = reverse_lazy("accounts:settings") - - def get(self, request, *args, **kwargs): - response = super().get(request, *args, **kwargs) - - user = request.user - task_active = cache.get(f"{user.email}-reddit-refresh") - - if not task_active: - RedditTokenTask.delay(user.pk) - messages.success(request, _("Access token is being retrieved")) - cache.set(f"{user.email}-reddit-refresh", 1, 300) - return response - - messages.error(request, _("Unable to retrieve token")) - return response diff --git a/src/newsreader/accounts/views/__init__.py b/src/newsreader/accounts/views/__init__.py new file mode 100644 index 0000000..81dd1fc --- /dev/null +++ b/src/newsreader/accounts/views/__init__.py @@ -0,0 +1,26 @@ +from newsreader.accounts.views.auth import LoginView, LogoutView +from newsreader.accounts.views.integrations import ( + IntegrationsView, + RedditRevokeRedirectView, + RedditTemplateView, + RedditTokenRedirectView, + TwitterAuthRedirectView, + TwitterRevokeRedirectView, + TwitterTemplateView, +) +from newsreader.accounts.views.password import ( + PasswordChangeView, + PasswordResetCompleteView, + PasswordResetConfirmView, + PasswordResetDoneView, + PasswordResetView, +) +from newsreader.accounts.views.registration import ( + ActivationCompleteView, + ActivationResendView, + ActivationView, + RegistrationClosedView, + RegistrationCompleteView, + RegistrationView, +) +from newsreader.accounts.views.settings import SettingsView diff --git a/src/newsreader/accounts/views/auth.py b/src/newsreader/accounts/views/auth.py new file mode 100644 index 0000000..0663768 --- /dev/null +++ b/src/newsreader/accounts/views/auth.py @@ -0,0 +1,11 @@ +from django.contrib.auth import views as django_views +from django.urls import reverse_lazy + + +class LoginView(django_views.LoginView): + template_name = "accounts/views/login.html" + success_url = reverse_lazy("index") + + +class LogoutView(django_views.LogoutView): + next_page = reverse_lazy("accounts:login") diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py new file mode 100644 index 0000000..62d71fc --- /dev/null +++ b/src/newsreader/accounts/views/integrations.py @@ -0,0 +1,343 @@ +import logging + +from urllib.parse import parse_qs, urlencode + +from django.conf import settings +from django.contrib import messages +from django.core.cache import cache +from django.shortcuts import redirect +from django.urls import reverse_lazy +from django.utils.translation import gettext as _ +from django.views.generic import RedirectView, TemplateView + +from requests_oauthlib import OAuth1 as OAuth + +from newsreader.news.collection.exceptions import StreamException +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, + revoke_reddit_token, +) +from newsreader.news.collection.tasks import RedditTokenTask +from newsreader.news.collection.twitter import ( + TWITTER_ACCESS_TOKEN_URL, + TWITTER_AUTH_URL, + TWITTER_REQUEST_TOKEN_URL, + TWITTER_REVOKE_URL, +) +from newsreader.news.collection.utils import post + + +logger = logging.getLogger(__name__) + + +class IntegrationsView(TemplateView): + template_name = "accounts/views/integrations.html" + + def get_context_data(self, **kwargs): + return { + **super().get_context_data(**kwargs), + **self.get_reddit_context(**kwargs), + **self.get_twitter_context(**kwargs), + } + + def get_reddit_context(self, **kwargs): + user = self.request.user + reddit_authorization_url = None + reddit_refresh_url = None + + reddit_task_active = cache.get(f"{user.email}-reddit-refresh") + + if ( + user.reddit_refresh_token + and not user.reddit_access_token + and not reddit_task_active + ): + reddit_refresh_url = reverse_lazy("accounts:reddit-refresh") + + if not user.reddit_refresh_token: + reddit_authorization_url = get_reddit_authorization_url(user) + + return { + "reddit_authorization_url": reddit_authorization_url, + "reddit_refresh_url": reddit_refresh_url, + "reddit_revoke_url": ( + reverse_lazy("accounts:reddit-revoke") + if not reddit_authorization_url + else None + ), + } + + def get_twitter_context(self, **kwargs): + twitter_revoke_url = None + + if self.request.user.has_twitter_auth: + twitter_revoke_url = reverse_lazy("accounts:twitter-revoke") + + return { + "twitter_auth_url": reverse_lazy("accounts:twitter-auth"), + "twitter_revoke_url": twitter_revoke_url, + } + + +class RedditTemplateView(TemplateView): + template_name = "accounts/views/reddit.html" + + def get(self, request, *args, **kwargs): + context = self.get_context_data(**kwargs) + + error = request.GET.get("error", None) + state = request.GET.get("state", None) + code = request.GET.get("code", None) + + if error: + return self.render_to_response({**context, "error": error}) + + if not code or not state: + return self.render_to_response(context) + + cached_state = cache.get(f"{request.user.email}-reddit-auth") + + if state != cached_state: + return self.render_to_response( + { + **context, + "error": _( + "The saved state for Reddit authorization did not match" + ), + } + ) + + try: + access_token, refresh_token = get_reddit_access_token(code, request.user) + + return self.render_to_response( + { + **context, + "access_token": access_token, + "refresh_token": refresh_token, + } + ) + except StreamException as e: + return self.render_to_response({**context, "error": str(e)}) + except KeyError: + return self.render_to_response( + { + **context, + "error": _("Access and refresh token not found in response"), + } + ) + + +class RedditTokenRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + response = super().get(request, *args, **kwargs) + + user = request.user + task_active = cache.get(f"{user.email}-reddit-refresh") + + if not task_active: + RedditTokenTask.delay(user.pk) + messages.success(request, _("Access token is being retrieved")) + cache.set(f"{user.email}-reddit-refresh", 1, 300) + return response + + messages.error(request, _("Unable to retrieve token")) + return response + + +class RedditRevokeRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + response = super().get(request, *args, **kwargs) + + user = request.user + + if not user.reddit_refresh_token: + messages.error(request, _("No reddit account is linked to this account")) + return response + + try: + is_revoked = revoke_reddit_token(user) + except StreamException: + logger.exception(f"Unable to revoke reddit token for {user.pk}") + + messages.error(request, _("Unable to revoke reddit token")) + return response + + if not is_revoked: + messages.error(request, _("Unable to revoke reddit token")) + return response + + user.reddit_access_token = None + user.reddit_refresh_token = None + user.save() + + messages.success(request, _("Reddit account deathorized")) + return response + + +class TwitterRevokeRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + if not request.user.has_twitter_auth: + messages.error(request, _("No twitter credentials found")) + return super().get(request, *args, **kwargs) + + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=request.user.twitter_oauth_token, + resource_owner_secret=request.user.twitter_oauth_token_secret, + ) + + try: + post(TWITTER_REVOKE_URL, auth=oauth) + except StreamException: + logger.exception("Failed revoking Twitter account") + + messages.error(request, _("Unable revoke Twitter account")) + return super().get(request, *args, **kwargs) + + request.user.twitter_oauth_token = None + request.user.twitter_oauth_token_secret = None + request.user.save() + + messages.success(request, _("Twitter account revoked")) + return super().get(request, *args, **kwargs) + + +class TwitterAuthRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + callback_uri=settings.TWITTER_REDIRECT_URL, + ) + + try: + response = post(TWITTER_REQUEST_TOKEN_URL, auth=oauth) + except StreamException: + logger.exception("Failed requesting Twitter authentication token") + + messages.error(request, _("Unable to retrieve initial Twitter token")) + return super().get(request, *args, **kwargs) + + params = parse_qs(response.text) + + try: + request_oauth_token = params["oauth_token"][0] + request_oauth_secret = params["oauth_token_secret"][0] + except KeyError: + logger.exception("No credentials found in response") + + messages.error(request, _("Unable to retrieve initial Twitter token")) + return super().get(request, *args, **kwargs) + + cache.set_many( + { + f"twitter-{request.user.email}-token": request_oauth_token, + f"twitter-{request.user.email}-secret": request_oauth_secret, + } + ) + + request_params = urlencode({"oauth_token": request_oauth_token}) + return redirect(f"{TWITTER_AUTH_URL}/?{request_params}") + + +class TwitterTemplateView(TemplateView): + template_name = "accounts/views/twitter.html" + + def get(self, request, *args, **kwargs): + context = self.get_context_data(**kwargs) + + denied = request.GET.get("denied", False) + oauth_token = request.GET.get("oauth_token") + oauth_verifier = request.GET.get("oauth_verifier") + + if denied: + return self.render_to_response( + { + **context, + "error": _("Twitter authorization failed"), + "authorized": False, + } + ) + + cached_token = cache.get(f"twitter-{request.user.email}-token") + + if oauth_token != cached_token: + return self.render_to_response( + { + **context, + "error": _("OAuth tokens failed to match"), + "authorized": False, + } + ) + + cached_secret = cache.get(f"twitter-{request.user.email}-secret") + + if not cached_token or not cached_secret: + return self.render_to_response( + { + **context, + "error": _("No matching tokens found for this user"), + "authorized": False, + } + ) + + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=cached_token, + resource_owner_secret=cached_secret, + verifier=oauth_verifier, + ) + + try: + response = post(TWITTER_ACCESS_TOKEN_URL, auth=oauth) + except StreamException: + logger.exception("Failed requesting Twitter access token") + + return self.render_to_response( + { + **context, + "error": _("Failed requesting access token"), + "authorized": False, + } + ) + + params = parse_qs(response.text) + + try: + oauth_token = params["oauth_token"][0] + oauth_secret = params["oauth_token_secret"][0] + except KeyError: + logger.exception("No credentials in Twitter response") + + return self.render_to_response( + { + **context, + "error": _("No credentials found in Twitter response"), + "authorized": False, + } + ) + + request.user.twitter_oauth_token = oauth_token + request.user.twitter_oauth_token_secret = oauth_secret + request.user.save() + + cache.delete_many( + [ + f"twitter-{request.user.email}-token", + f"twitter-{request.user.email}-secret", + ] + ) + + return self.render_to_response({**context, "error": None, "authorized": True}) diff --git a/src/newsreader/accounts/views/password.py b/src/newsreader/accounts/views/password.py new file mode 100644 index 0000000..e9e0aa3 --- /dev/null +++ b/src/newsreader/accounts/views/password.py @@ -0,0 +1,37 @@ +from django.contrib.auth import views as django_views +from django.urls import reverse_lazy + +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) + + +# PasswordResetView sends the mail +# PasswordResetDoneView shows a success message for the above +# PasswordResetConfirmView checks the link the user clicked and +# prompts for a new password +# PasswordResetCompleteView shows a success message for the above +class PasswordResetView(django_views.PasswordResetView): + template_name = "password-reset/password-reset.html" + subject_template_name = "password-reset/password-reset-subject.txt" + email_template_name = "password-reset/password-reset-email.html" + success_url = reverse_lazy("accounts:password-reset-done") + + +class PasswordResetDoneView(django_views.PasswordResetDoneView): + template_name = "password-reset/password-reset-done.html" + + +class PasswordResetConfirmView(django_views.PasswordResetConfirmView): + template_name = "password-reset/password-reset-confirm.html" + success_url = reverse_lazy("accounts:password-reset-complete") + + +class PasswordResetCompleteView(django_views.PasswordResetCompleteView): + template_name = "password-reset/password-reset-complete.html" + + +class PasswordChangeView(django_views.PasswordChangeView): + template_name = "accounts/views/password-change.html" + success_url = reverse_lazy("accounts:settings") diff --git a/src/newsreader/accounts/views/registration.py b/src/newsreader/accounts/views/registration.py new file mode 100644 index 0000000..597aa9a --- /dev/null +++ b/src/newsreader/accounts/views/registration.py @@ -0,0 +1,59 @@ +from django.shortcuts import render +from django.urls import reverse_lazy +from django.views.generic import TemplateView + +from registration.backends.default import views as registration_views + +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) + + +# RegistrationView shows a registration form and sends the email +# RegistrationCompleteView shows after filling in the registration form +# ActivationView is send within the activation email and activates the account +# ActivationCompleteView shows the success screen when activation was succesful +# ActivationResendView can be used when activation links are expired +# RegistrationClosedView shows when registration is disabled +class RegistrationView(registration_views.RegistrationView): + disallowed_url = reverse_lazy("accounts:register-closed") + template_name = "registration/registration_form.html" + success_url = reverse_lazy("accounts:register-complete") + + +class RegistrationCompleteView(TemplateView): + template_name = "registration/registration_complete.html" + + +class RegistrationClosedView(TemplateView): + template_name = "registration/registration_closed.html" + + +# Redirects or renders failed activation template +class ActivationView(registration_views.ActivationView): + template_name = "registration/activation_failure.html" + + def get_success_url(self, user): + return ("accounts:activate-complete", (), {}) + + +class ActivationCompleteView(TemplateView): + template_name = "registration/activation_complete.html" + + +# Renders activation form resend or resend_activation_complete +class ActivationResendView(registration_views.ResendActivationView): + template_name = "registration/activation_resend_form.html" + + def render_form_submitted_template(self, form): + """ + Renders resend activation complete template with the submitted email. + + """ + email = form.cleaned_data["email"] + context = {"email": email} + + return render( + self.request, "registration/activation_resend_complete.html", context + ) diff --git a/src/newsreader/accounts/views/settings.py b/src/newsreader/accounts/views/settings.py new file mode 100644 index 0000000..1603252 --- /dev/null +++ b/src/newsreader/accounts/views/settings.py @@ -0,0 +1,26 @@ +from django.urls import reverse_lazy +from django.views.generic.edit import FormView, ModelFormMixin + +from newsreader.accounts.forms import UserSettingsForm +from newsreader.accounts.models import User +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) + + +class SettingsView(ModelFormMixin, FormView): + template_name = "accounts/views/settings.html" + success_url = reverse_lazy("accounts:settings") + form_class = UserSettingsForm + model = User + + def get(self, request, *args, **kwargs): + self.object = self.get_object() + return super().get(request, *args, **kwargs) + + def get_object(self, **kwargs): + return self.request.user + + def get_form_kwargs(self): + return {**super().get_form_kwargs(), "instance": self.request.user} diff --git a/src/newsreader/conf/base.py b/src/newsreader/conf/base.py index 43b89fd..d41f352 100644 --- a/src/newsreader/conf/base.py +++ b/src/newsreader/conf/base.py @@ -129,19 +129,14 @@ LOGGING = { "class": "logging.StreamHandler", "formatter": "timestamped", }, - "mail_admins": { - "level": "ERROR", - "filters": ["require_debug_false"], - "class": "django.utils.log.AdminEmailHandler", - }, - "syslog": { + "celery": { "level": "INFO", "filters": ["require_debug_false"], "class": "logging.handlers.SysLogHandler", "formatter": "syslog", "address": "/dev/log", }, - "syslog_errors": { + "syslog": { "level": "ERROR", "filters": ["require_debug_false"], "class": "logging.handlers.SysLogHandler", @@ -150,26 +145,13 @@ LOGGING = { }, }, "loggers": { - "django": { - "handlers": ["console", "mail_admins", "syslog_errors"], - "level": "WARNING", - }, + "django": {"handlers": ["console", "syslog"], "level": "INFO"}, "django.server": { - "handlers": ["console", "syslog_errors"], - "level": "INFO", - "propagate": False, - }, - "django.request": { - "handlers": ["console", "syslog_errors"], - "level": "INFO", - "propagate": False, - }, - "celery": {"handlers": ["syslog", "console"], "level": "INFO"}, - "celery.task": { - "handlers": ["syslog", "console"], + "handlers": ["console", "syslog"], "level": "INFO", "propagate": False, }, + "celery": {"handlers": ["celery", "console"], "level": "INFO"}, "newsreader": {"handlers": ["syslog", "console"], "level": "INFO"}, }, } @@ -219,7 +201,16 @@ VERSION = get_current_version() # Reddit integration REDDIT_CLIENT_ID = "CLIENT_ID" REDDIT_CLIENT_SECRET = "CLIENT_SECRET" -REDDIT_REDIRECT_URL = "http://127.0.0.1:8000/accounts/settings/reddit/callback/" +REDDIT_REDIRECT_URL = ( + "http://127.0.0.1:8000/accounts/settings/integrations/reddit/callback/" +) + +# Twitter integration +TWITTER_CONSUMER_ID = "CONSUMER_ID" +TWITTER_CONSUMER_SECRET = "CONSUMER_SECRET" +TWITTER_REDIRECT_URL = ( + "http://127.0.0.1:8000/accounts/settings/integrations/twitter/callback/" +) # Third party settings AXES_HANDLER = "axes.handlers.cache.AxesCacheHandler" diff --git a/src/newsreader/conf/production.py b/src/newsreader/conf/production.py index bfe9818..f481885 100644 --- a/src/newsreader/conf/production.py +++ b/src/newsreader/conf/production.py @@ -46,9 +46,14 @@ TEMPLATES = [ ] # Reddit integration -REDDIT_CLIENT_ID = os.environ["REDDIT_CLIENT_ID"] -REDDIT_CLIENT_SECRET = os.environ["REDDIT_CLIENT_SECRET"] -REDDIT_REDIRECT_URL = os.environ["REDDIT_CALLBACK_URL"] +REDDIT_CLIENT_ID = os.environ.get("REDDIT_CLIENT_ID", "") +REDDIT_CLIENT_SECRET = os.environ.get("REDDIT_CLIENT_SECRET", "") +REDDIT_REDIRECT_URL = os.environ.get("REDDIT_CALLBACK_URL", "") + +# Twitter integration +TWITTER_CONSUMER_ID = os.environ.get("TWITTER_CONSUMER_ID", "") +TWITTER_CONSUMER_SECRET = os.environ.get("TWITTER_CONSUMER_SECRET", "") +TWITTER_REDIRECT_URL = os.environ.get("TWITTER_REDIRECT_URL", "") # Third party settings AXES_HANDLER = "axes.handlers.database.AxesDatabaseHandler" diff --git a/src/newsreader/fixtures/default-fixture.json b/src/newsreader/fixtures/default-fixture.json index 10d6416..1794742 100644 --- a/src/newsreader/fixtures/default-fixture.json +++ b/src/newsreader/fixtures/default-fixture.json @@ -1,4023 +1,4023 @@ -[ -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "admin", - "model": "logentry" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "auth", - "model": "permission" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "auth", - "model": "group" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "contenttypes", - "model": "contenttype" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "sessions", - "model": "session" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "crontabschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "intervalschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "periodictask" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "periodictasks" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "solarschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "clockedschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "registration", - "model": "registrationprofile" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "registration", - "model": "supervisedregistrationprofile" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "axes", - "model": "accessattempt" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "axes", - "model": "accesslog" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "accounts", - "model": "user" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "core", - "model": "post" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "core", - "model": "category" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "collection", - "model": "collectionrule" - } -}, -{ - "model": "sessions.session", - "pk": "3sumq22krk8tsvexcs4b8czu82yhvuer", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-05-16T18:29:04.049Z" - } -}, -{ - "model": "sessions.session", - "pk": "8ix6bdwf2ywk0eir1hb062dhfh9xit85", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-07-21T19:36:54.530Z" - } -}, -{ - "model": "sessions.session", - "pk": "d4wophwpjm8z96doe8iddvhdv9yfafyx", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-06-07T19:45:49.727Z" - } -}, -{ - "model": "sessions.session", - "pk": "g23ziz66li5zx8nd8cewb3vevdxhjkm0", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-06-30T06:55:50.747Z" - } -}, -{ - "model": "sessions.session", - "pk": "jwn66dptmdkm6hom2ns3j288aaxqtyjd", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-06-07T18:38:19.116Z" - } -}, -{ - "model": "sessions.session", - "pk": "wjz6kwg5e5ciemre0l0wwyrcwcj2gyg6", - "fields": { - "session_data": "MWU5ODBjY2QyOTFhMmRiY2QyYjQwZjQ3MmMwYmExYjBlYTkxNTcwODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiI0YWZkYTkxNzU5ZDBhZDZmMjg1ZTQyOGY0OTUxN2M5MTJhMmM5NWIyIn0=", - "expire_date": "2020-08-09T09:52:04.705Z" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 1, - "fields": { - "every": 5, - "period": "minutes" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 2, - "fields": { - "every": 15, - "period": "minutes" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 3, - "fields": { - "every": 30, - "period": "minutes" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 4, - "fields": { - "every": 1, - "period": "hours" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 5, - "fields": { - "every": 4, - "period": "hours" - } -}, -{ - "model": "django_celery_beat.crontabschedule", - "pk": 1, - "fields": { - "minute": "0", - "hour": "4", - "day_of_week": "*", - "day_of_month": "*", - "month_of_year": "*", - "timezone": "UTC" - } -}, -{ - "model": "django_celery_beat.periodictasks", - "pk": 1, - "fields": { - "last_update": "2020-07-26T09:47:48.298Z" - } -}, -{ - "model": "django_celery_beat.periodictask", - "pk": 1, - "fields": { - "name": "celery.backend_cleanup", - "task": "celery.backend_cleanup", - "interval": null, - "crontab": 1, - "solar": null, - "clocked": null, - "args": "[]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "headers": "{}", - "priority": null, - "expires": null, - "expire_seconds": 43200, - "one_off": false, - "start_time": null, - "enabled": true, - "last_run_at": "2020-07-26T09:47:48.322Z", - "total_run_count": 17, - "date_changed": "2020-07-26T09:47:50.362Z", - "description": "" - } -}, -{ - "model": "django_celery_beat.periodictask", - "pk": 10, - "fields": { - "name": "sonny@bakker.nl-collection-task", - "task": "FeedTask", - "interval": 5, - "crontab": null, - "solar": null, - "clocked": null, - "args": "[1]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "headers": "{}", - "priority": null, - "expires": null, - "expire_seconds": null, - "one_off": false, - "start_time": null, - "enabled": false, - "last_run_at": "2020-07-14T11:45:26.209Z", - "total_run_count": 307, - "date_changed": "2020-07-14T11:45:41.282Z", - "description": "" - } -}, -{ - "model": "django_celery_beat.periodictask", - "pk": 11, - "fields": { - "name": "Reddit collection task", - "task": "RedditTask", - "interval": 5, - "crontab": null, - "solar": null, - "clocked": null, - "args": "[]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "headers": "{}", - "priority": null, - "expires": null, - "expire_seconds": null, - "one_off": false, - "start_time": null, - "enabled": false, - "last_run_at": null, - "total_run_count": 4, - "date_changed": "2020-07-14T11:45:41.316Z", - "description": "" - } -}, -{ - "model": "core.post", - "pk": 3061, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:14:50.423Z", - "title": "Star Citizen: Question and Answer Thread", - "body": "

Welcome to the Star Citizen question and answer thread. Feel free to ask any questions you have related to SC here!

\n\n\n\n

Useful Links and Resources:

\n\n

Star Citizen Wiki - The biggest and best wiki resource dedicated to Star Citizen

\n\n

Star Citizen FAQ - Chances the answer you need is here.

\n\n

Discord Help Channel - Often times community members will be here to help you with issues.

\n\n

Referral Code Randomizer - Use this when creating a new account to get 5000 extra UEC.

\n\n

Download Star Citizen - Get the latest version of Star Citizen here

\n\n

Current Game Features - Click here to see what you can currently do in Star Citizen.

\n\n

Development Roadmap - The current development status of up and coming Star Citizen features.

\n\n

Pledge FAQ - Official FAQ regarding spending money on the game.

\n
", - "author": "UEE_Central_Computer", - "publication_date": "2020-07-20T14:00:10Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huk04t/star_citizen_question_and_answer_thread/", - "read": false, - "rule": 82, - "remote_identifier": "huk04t" - } -}, -{ - "model": "core.post", - "pk": 3062, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:33:37.019Z", - "title": "Peace and Quiet", - "body": "
\"Peace
", - "author": "SourMemeNZ", - "publication_date": "2020-07-20T14:09:49Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huk4ib/peace_and_quiet/", - "read": true, - "rule": 82, - "remote_identifier": "huk4ib" - } -}, -{ - "model": "core.post", - "pk": 3063, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:14:50.463Z", - "title": "Y'all are probably sick of em by now but here's my LEGO Mercury Star Runner (MSR).", - "body": "
\"Y'all
", - "author": "osamadabinman", - "publication_date": "2020-07-20T19:53:23Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hupzqa/yall_are_probably_sick_of_em_by_now_but_heres_my/", - "read": true, - "rule": 82, - "remote_identifier": "hupzqa" - } -}, -{ - "model": "core.post", - "pk": 3064, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:12.253Z", - "title": "Damned Space Invaders and their pixel weapons!", - "body": "
\"Damned
", - "author": "Akaradrin", - "publication_date": "2020-07-20T14:26:18Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hukckf/damned_space_invaders_and_their_pixel_weapons/", - "read": true, - "rule": 82, - "remote_identifier": "hukckf" - } -}, -{ - "model": "core.post", - "pk": 3065, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.578Z", - "title": "The sky is no longer the limit", - "body": "
\"The
", - "author": "CyberTill", - "publication_date": "2020-07-20T14:11:31Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huk5b8/the_sky_is_no_longer_the_limit/", - "read": false, - "rule": 82, - "remote_identifier": "huk5b8" - } -}, -{ - "model": "core.post", - "pk": 3066, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:23.282Z", - "title": "Terrapin Hover Mode Gameplay [Full Video in Comments]", - "body": "
", - "author": "Didactic_Tomato", - "publication_date": "2020-07-20T11:01:13Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hui1gv/terrapin_hover_mode_gameplay_full_video_in/", - "read": true, - "rule": 82, - "remote_identifier": "hui1gv" - } -}, -{ - "model": "core.post", - "pk": 3067, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:44.250Z", - "title": "honestly", - "body": "
\"honestly\"
", - "author": "Beatlead", - "publication_date": "2020-07-20T18:24:07Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huo96t/honestly/", - "read": true, - "rule": 82, - "remote_identifier": "huo96t" - } -}, -{ - "model": "core.post", - "pk": 3068, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.584Z", - "title": "As a paranoiac and tired of checking if door was closed, saved to f4 thoses \"security cam\" positions, could be usefull for larger ships :)", - "body": "
Direct url
", - "author": "icwiener__", - "publication_date": "2020-07-20T13:03:33Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hujchz/as_a_paranoiac_and_tired_of_checking_if_door_was/", - "read": false, - "rule": 82, - "remote_identifier": "hujchz" - } -}, -{ - "model": "core.post", - "pk": 3069, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:33:59.158Z", - "title": "Station Manager: \"You're too fat, we won't let you in, go and fall on Lorville. Thank you for your call!\" Me: \"okay :'(\"", - "body": "
\"Station
", - "author": "Shaman_N_One", - "publication_date": "2020-07-20T11:33:38Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huidlu/station_manager_youre_too_fat_we_wont_let_you_in/", - "read": true, - "rule": 82, - "remote_identifier": "huidlu" - } -}, -{ - "model": "core.post", - "pk": 3070, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.588Z", - "title": "[PTU Bug Hunt Request] Packet Loss", - "body": "
Direct url
", - "author": "Rainwalker007", - "publication_date": "2020-07-20T18:38:03Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huoicq/ptu_bug_hunt_request_packet_loss/", - "read": false, - "rule": 82, - "remote_identifier": "huoicq" - } -}, -{ - "model": "core.post", - "pk": 3071, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:52.092Z", - "title": "Anyone able to explain these \"trail frames\"?", - "body": "
\"Anyone
", - "author": "Abnormal_Sloth", - "publication_date": "2020-07-20T17:11:32Z", - "url": "https://www.reddit.com/r/starcitizen/comments/humyeq/anyone_able_to_explain_these_trail_frames/", - "read": true, - "rule": 82, - "remote_identifier": "humyeq" - } -}, -{ - "model": "core.post", - "pk": 3072, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.593Z", - "title": "#BringBackBugSmasher - A long forgotten legendary video content", - "body": "
Direct url
", - "author": "MasterBoring", - "publication_date": "2020-07-20T18:05:54Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hunx77/bringbackbugsmasher_a_long_forgotten_legendary/", - "read": false, - "rule": 82, - "remote_identifier": "hunx77" - } -}, -{ - "model": "core.post", - "pk": 3073, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:33:22.601Z", - "title": "Oracle Helmet [in-game screenshot; downsampled to 4k]", - "body": "
\"Oracle
", - "author": "mr-hasgaha", - "publication_date": "2020-07-20T17:39:34Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hung0b/oracle_helmet_ingame_screenshot_downsampled_to_4k/", - "read": true, - "rule": 82, - "remote_identifier": "hung0b" - } -}, -{ - "model": "core.post", - "pk": 3074, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:34:42.578Z", - "title": "Testing 3.10 - Gladius in decoupled mode", - "body": "
", - "author": "DarkConstant", - "publication_date": "2020-07-19T21:26:52Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hu6f1h/testing_310_gladius_in_decoupled_mode/", - "read": true, - "rule": 82, - "remote_identifier": "hu6f1h" - } -}, -{ - "model": "core.post", - "pk": 3075, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:34:29.424Z", - "title": "Day 3, I can't stop taking pictures with my Carrack. Send help", - "body": "
\"Day
", - "author": "CyberTill", - "publication_date": "2020-07-20T01:58:15Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huazyy/day_3_i_cant_stop_taking_pictures_with_my_carrack/", - "read": true, - "rule": 82, - "remote_identifier": "huazyy" - } -}, -{ - "model": "core.post", - "pk": 3076, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.602Z", - "title": "I used to enjoy flying between the buildings of new babbage, I mean before the NFZ \"improvement\"", - "body": "
\"I
", - "author": "shoeii", - "publication_date": "2020-07-20T16:40:26Z", - "url": "https://www.reddit.com/r/starcitizen/comments/humet2/i_used_to_enjoy_flying_between_the_buildings_of/", - "read": false, - "rule": 82, - "remote_identifier": "humet2" - } -}, -{ - "model": "core.post", - "pk": 3077, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:18:04.237Z", - "title": "Thank you CIG for updated heightmaps and render distances", - "body": "
\"Thank
", - "author": "u7f76", - "publication_date": "2020-07-19T23:38:22Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hu8pwf/thank_you_cig_for_updated_heightmaps_and_render/", - "read": true, - "rule": 82, - "remote_identifier": "hu8pwf" - } -}, -{ - "model": "core.post", - "pk": 3078, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.607Z", - "title": "This Week in Star Citizen | July 20th 2020", - "body": "
Direct url
", - "author": "ivtiprogamer", - "publication_date": "2020-07-20T19:50:29Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hupxnt/this_week_in_star_citizen_july_20th_2020/", - "read": false, - "rule": 82, - "remote_identifier": "hupxnt" - } -}, -{ - "model": "core.post", - "pk": 3079, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:34:36.068Z", - "title": "Bravo CIG lighting team! Noticeable improvements to all around environment lighting in 3.10", - "body": "
\"Bravo
", - "author": "u7f76", - "publication_date": "2020-07-20T00:02:23Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hu94o0/bravo_cig_lighting_team_noticeable_improvements/", - "read": true, - "rule": 82, - "remote_identifier": "hu94o0" - } -}, -{ - "model": "core.post", - "pk": 3080, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.613Z", - "title": "Thick", - "body": "
\"Thick\"
", - "author": "burgerbagel", - "publication_date": "2020-07-20T16:24:38Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hum50f/thick/", - "read": false, - "rule": 82, - "remote_identifier": "hum50f" - } -}, -{ - "model": "core.post", - "pk": 3081, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:34:19.763Z", - "title": "Soon\u2122", - "body": "
\"Soon\u2122\"
", - "author": "Mistralette", - "publication_date": "2020-07-20T05:54:09Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hueg01/soon/", - "read": true, - "rule": 82, - "remote_identifier": "hueg01" - } -}, -{ - "model": "core.post", - "pk": 3082, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.618Z", - "title": "On the prowl", - "body": "
\"On
", - "author": "SaraCaterina", - "publication_date": "2020-07-20T16:37:03Z", - "url": "https://www.reddit.com/r/starcitizen/comments/humcmb/on_the_prowl/", - "read": false, - "rule": 82, - "remote_identifier": "humcmb" - } -}, -{ - "model": "core.post", - "pk": 3083, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:34:07.272Z", - "title": "The Hills Have Eyes", - "body": "
\"The
", - "author": "FallenLordik", - "publication_date": "2020-07-20T11:19:19Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hui8ao/the_hills_have_eyes/", - "read": true, - "rule": 82, - "remote_identifier": "hui8ao" - } -}, -{ - "model": "core.post", - "pk": 3084, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.623Z", - "title": "Worried about longer loading screens? Hit ~ and do r_displayinfo 3", - "body": "
\"Worried
", - "author": "kristokn", - "publication_date": "2020-07-20T10:09:53Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huhif1/worried_about_longer_loading_screens_hit_and_do_r/", - "read": false, - "rule": 82, - "remote_identifier": "huhif1" - } -}, -{ - "model": "core.post", - "pk": 3085, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.625Z", - "title": "My contribution to the wallpaper contest... click for the full effect (3440x1440)", - "body": "
\"My
", - "author": "Dougie_Juice", - "publication_date": "2020-07-20T20:02:31Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huq655/my_contribution_to_the_wallpaper_contest_click/", - "read": false, - "rule": 82, - "remote_identifier": "huq655" - } -}, -{ - "model": "core.post", - "pk": 3086, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.627Z", - "title": "Star Citizen: The Onion (Parody Project)", - "body": "
Direct url
", - "author": "BroadOne", - "publication_date": "2020-07-20T19:19:20Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hupbkj/star_citizen_the_onion_parody_project/", - "read": false, - "rule": 82, - "remote_identifier": "hupbkj" - } -}, -{ - "model": "core.post", - "pk": 3087, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.637Z", - "title": "perfect day to sunbathe", - "body": "
", - "author": "Pedrica1", - "publication_date": "2020-07-20T18:08:17Z", - "url": "https://www.reddit.com/r/aww/comments/hunysb/perfect_day_to_sunbathe/", - "read": false, - "rule": 81, - "remote_identifier": "hunysb" - } -}, -{ - "model": "core.post", - "pk": 3088, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.639Z", - "title": "My dogs face when he sees I'm home", - "body": "
", - "author": "NewReddit_WhoDis", - "publication_date": "2020-07-20T16:45:21Z", - "url": "https://www.reddit.com/r/aww/comments/humhxa/my_dogs_face_when_he_sees_im_home/", - "read": false, - "rule": 81, - "remote_identifier": "humhxa" - } -}, -{ - "model": "core.post", - "pk": 3089, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.641Z", - "title": "Cow loves the scritch machine", - "body": "
", - "author": "Der_Ist", - "publication_date": "2020-07-20T17:36:16Z", - "url": "https://www.reddit.com/r/aww/comments/hundvo/cow_loves_the_scritch_machine/", - "read": false, - "rule": 81, - "remote_identifier": "hundvo" - } -}, -{ - "model": "core.post", - "pk": 3090, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.643Z", - "title": "Can I sit next to you ?", - "body": "
", - "author": "wheezy098", - "publication_date": "2020-07-20T17:55:10Z", - "url": "https://www.reddit.com/r/aww/comments/hunq5h/can_i_sit_next_to_you/", - "read": false, - "rule": 81, - "remote_identifier": "hunq5h" - } -}, -{ - "model": "core.post", - "pk": 3091, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.645Z", - "title": "IS THAT A CUSTOMER? flop flop flop flop .... \" Can I uhh... help you sir?\"", - "body": "
", - "author": "MBMV", - "publication_date": "2020-07-20T12:50:40Z", - "url": "https://www.reddit.com/r/aww/comments/huj7g3/is_that_a_customer_flop_flop_flop_flop_can_i_uhh/", - "read": false, - "rule": 81, - "remote_identifier": "huj7g3" - } -}, -{ - "model": "core.post", - "pk": 3092, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.647Z", - "title": "Good Boy turned Disney Princess", - "body": "
", - "author": "Sauwercraud", - "publication_date": "2020-07-20T18:40:05Z", - "url": "https://www.reddit.com/r/aww/comments/huojq0/good_boy_turned_disney_princess/", - "read": false, - "rule": 81, - "remote_identifier": "huojq0" - } -}, -{ - "model": "core.post", - "pk": 3093, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.649Z", - "title": "Kitty loop", - "body": "
", - "author": "Dlatrex", - "publication_date": "2020-07-20T12:54:02Z", - "url": "https://www.reddit.com/r/aww/comments/huj8s6/kitty_loop/", - "read": false, - "rule": 81, - "remote_identifier": "huj8s6" - } -}, -{ - "model": "core.post", - "pk": 3094, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.652Z", - "title": "if i fits i sits", - "body": "
", - "author": "jasontaken", - "publication_date": "2020-07-20T16:38:32Z", - "url": "https://www.reddit.com/r/aww/comments/humdlf/if_i_fits_i_sits/", - "read": false, - "rule": 81, - "remote_identifier": "humdlf" - } -}, -{ - "model": "core.post", - "pk": 3095, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.654Z", - "title": "Isn\u2019t she Adorable !", - "body": "
\"Isn\u2019t
", - "author": "MunchyMac", - "publication_date": "2020-07-20T16:18:05Z", - "url": "https://www.reddit.com/r/aww/comments/hum133/isnt_she_adorable/", - "read": false, - "rule": 81, - "remote_identifier": "hum133" - } -}, -{ - "model": "core.post", - "pk": 3096, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.655Z", - "title": "Thank you mama (\u2283\uff61\u2022\u0301\u203f\u2022\u0300\uff61)\u2283", - "body": "
", - "author": "AnoushkaSingh", - "publication_date": "2020-07-20T13:35:51Z", - "url": "https://www.reddit.com/r/aww/comments/hujpxy/thank_you_mama/", - "read": false, - "rule": 81, - "remote_identifier": "hujpxy" - } -}, -{ - "model": "core.post", - "pk": 3097, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.657Z", - "title": "I WANT TO HUG HIM SO BAD!!!", - "body": "
", - "author": "BATMAN_5777", - "publication_date": "2020-07-20T18:25:20Z", - "url": "https://www.reddit.com/r/aww/comments/huo9z4/i_want_to_hug_him_so_bad/", - "read": false, - "rule": 81, - "remote_identifier": "huo9z4" - } -}, -{ - "model": "core.post", - "pk": 3098, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.659Z", - "title": "Before and after being called a good boy", - "body": "
\"Before
", - "author": "vladgrinch", - "publication_date": "2020-07-20T10:48:40Z", - "url": "https://www.reddit.com/r/aww/comments/huhwu9/before_and_after_being_called_a_good_boy/", - "read": false, - "rule": 81, - "remote_identifier": "huhwu9" - } -}, -{ - "model": "core.post", - "pk": 3099, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.662Z", - "title": "My fianc\u00e9 has wanted a dog his whole life. This is his college graduation present. Welcome home Maple!", - "body": "
\"My
", - "author": "AlexisaurusRex", - "publication_date": "2020-07-20T17:57:25Z", - "url": "https://www.reddit.com/r/aww/comments/hunrie/my_fianc\u00e9_has_wanted_a_dog_his_whole_life_this_is/", - "read": false, - "rule": 81, - "remote_identifier": "hunrie" - } -}, -{ - "model": "core.post", - "pk": 3100, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.664Z", - "title": "Cute burro.", - "body": "
\"Cute
", - "author": "Craftmine101", - "publication_date": "2020-07-20T13:45:32Z", - "url": "https://www.reddit.com/r/aww/comments/huju40/cute_burro/", - "read": false, - "rule": 81, - "remote_identifier": "huju40" - } -}, -{ - "model": "core.post", - "pk": 3101, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.666Z", - "title": "I've never seen anyone dance better than that turtle.", - "body": "
", - "author": "Ashley1023", - "publication_date": "2020-07-20T18:07:30Z", - "url": "https://www.reddit.com/r/aww/comments/hunya8/ive_never_seen_anyone_dance_better_than_that/", - "read": false, - "rule": 81, - "remote_identifier": "hunya8" - } -}, -{ - "model": "core.post", - "pk": 3102, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.669Z", - "title": "Someone\u2019s going to be quite surprised when he realizes all this new stuff isn\u2019t for him!", - "body": "
\"Someone\u2019s
", - "author": "molly590", - "publication_date": "2020-07-20T15:46:21Z", - "url": "https://www.reddit.com/r/aww/comments/hulikg/someones_going_to_be_quite_surprised_when_he/", - "read": false, - "rule": 81, - "remote_identifier": "hulikg" - } -}, -{ - "model": "core.post", - "pk": 3103, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.671Z", - "title": "my aunt asked me to paint her puppy and I think it turned out so cute!!!", - "body": "
\"my
", - "author": "PineappleLightt", - "publication_date": "2020-07-20T16:39:37Z", - "url": "https://www.reddit.com/r/aww/comments/humea0/my_aunt_asked_me_to_paint_her_puppy_and_i_think/", - "read": false, - "rule": 81, - "remote_identifier": "humea0" - } -}, -{ - "model": "core.post", - "pk": 3104, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.673Z", - "title": "Master Assassin", - "body": "
\"Master
", - "author": "LauWalker", - "publication_date": "2020-07-20T18:47:52Z", - "url": "https://www.reddit.com/r/aww/comments/huop8a/master_assassin/", - "read": false, - "rule": 81, - "remote_identifier": "huop8a" - } -}, -{ - "model": "core.post", - "pk": 3105, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.675Z", - "title": "Every time this tank cleaner cleans out the aquarium, this fish swims over to him looking for pets", - "body": "
Direct url
", - "author": "unnaturalorder", - "publication_date": "2020-07-20T05:29:30Z", - "url": "https://www.reddit.com/r/aww/comments/hue3r0/every_time_this_tank_cleaner_cleans_out_the/", - "read": false, - "rule": 81, - "remote_identifier": "hue3r0" - } -}, -{ - "model": "core.post", - "pk": 3106, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.678Z", - "title": "My girlfriend sent me this while I was at work. And here I was thinking the perfect picture of our dog didn't exist", - "body": "
Direct url
", - "author": "Khuma-zi_Eldrama", - "publication_date": "2020-07-20T19:22:48Z", - "url": "https://www.reddit.com/r/aww/comments/hupdz8/my_girlfriend_sent_me_this_while_i_was_at_work/", - "read": false, - "rule": 81, - "remote_identifier": "hupdz8" - } -}, -{ - "model": "core.post", - "pk": 3107, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.680Z", - "title": "My first ever post, everyone meet my new baby girl Kiora! I\u2019m so in love with her\ud83e\udd7a\ud83d\udcab", - "body": "
\"My
", - "author": "Dumpling2463", - "publication_date": "2020-07-20T05:34:29Z", - "url": "https://www.reddit.com/r/aww/comments/hue6dx/my_first_ever_post_everyone_meet_my_new_baby_girl/", - "read": false, - "rule": 81, - "remote_identifier": "hue6dx" - } -}, -{ - "model": "core.post", - "pk": 3108, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.682Z", - "title": "Dog splashing in water", - "body": "
Direct url
", - "author": "TheRikari", - "publication_date": "2020-07-20T15:44:02Z", - "url": "https://www.reddit.com/r/aww/comments/hulh8k/dog_splashing_in_water/", - "read": false, - "rule": 81, - "remote_identifier": "hulh8k" - } -}, -{ - "model": "core.post", - "pk": 3109, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.685Z", - "title": "They say taking breaks is the key to productivity!", - "body": "
", - "author": "Thereaper29", - "publication_date": "2020-07-20T05:43:40Z", - "url": "https://www.reddit.com/r/aww/comments/hueawt/they_say_taking_breaks_is_the_key_to_productivity/", - "read": false, - "rule": 81, - "remote_identifier": "hueawt" - } -}, -{ - "model": "core.post", - "pk": 3110, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.687Z", - "title": "I went away for 3 weeks, and now my cat is in love with my husband", - "body": "
\"I
", - "author": "sillykittyish", - "publication_date": "2020-07-20T03:29:11Z", - "url": "https://www.reddit.com/r/aww/comments/hucd7u/i_went_away_for_3_weeks_and_now_my_cat_is_in_love/", - "read": false, - "rule": 81, - "remote_identifier": "hucd7u" - } -}, -{ - "model": "core.post", - "pk": 3111, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.689Z", - "title": "Can you feel the love", - "body": "
", - "author": "kettySewrdPic", - "publication_date": "2020-07-20T09:13:32Z", - "url": "https://www.reddit.com/r/aww/comments/hugx1k/can_you_feel_the_love/", - "read": false, - "rule": 81, - "remote_identifier": "hugx1k" - } -}, -{ - "model": "core.post", - "pk": 3112, - "fields": { - "created": "2020-07-20T19:32:35.835Z", - "modified": "2020-07-21T20:14:50.522Z", - "title": "Linux Experiences/Rants or Education/Certifications thread - July 20, 2020", - "body": "

Welcome to r/linux rants and experiences! This megathread is also to hear opinions from anyone just starting out with Linux or those that have used Linux (GNU or otherwise) for a long time.

\n\n

Let us know what's annoying you, whats making you happy, or something that you want to get out to r/linux but didn't make the cut into a full post of it's own.

\n\n

For those looking for certifications please use this megathread to ask about how to get certified whether it's for the business world or for your own satisfaction. Be sure to check out r/linuxadmin for more discussion in the SysAdmin world!

\n\n

Please keep questions in r/linuxquestions, r/linux4noobs, or the Wednesday automod thread.

\n
", - "author": "AutoModerator", - "publication_date": "2020-07-20T06:12:00Z", - "url": "https://www.reddit.com/r/linux/comments/hueoo0/linux_experiencesrants_or_educationcertifications/", - "read": false, - "rule": 80, - "remote_identifier": "hueoo0" - } -}, -{ - "model": "core.post", - "pk": 3113, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:19:49.339Z", - "title": "Unix Family Tree", - "body": "
\"Unix
", - "author": "bauripalash", - "publication_date": "2020-07-20T10:32:15Z", - "url": "https://www.reddit.com/r/linux/comments/huhqrh/unix_family_tree/", - "read": true, - "rule": 80, - "remote_identifier": "huhqrh" - } -}, -{ - "model": "core.post", - "pk": 3114, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:14:50.554Z", - "title": "NVIDIA open sourced part of NVAPI SDK to aid 'Windows emulation environments'", - "body": "
Direct url
", - "author": "ignapk", - "publication_date": "2020-07-20T13:17:19Z", - "url": "https://www.reddit.com/r/linux/comments/huji8c/nvidia_open_sourced_part_of_nvapi_sdk_to_aid/", - "read": false, - "rule": 80, - "remote_identifier": "huji8c" - } -}, -{ - "model": "core.post", - "pk": 3115, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:14:50.551Z", - "title": "Jellyfin 10.6 released", - "body": "
Direct url
", - "author": "resoluti0n_", - "publication_date": "2020-07-20T16:40:05Z", - "url": "https://www.reddit.com/r/linux/comments/humekr/jellyfin_106_released/", - "read": false, - "rule": 80, - "remote_identifier": "humekr" - } -}, -{ - "model": "core.post", - "pk": 3116, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:14:50.583Z", - "title": "[German] Article in major german newspaper about trying Linux and WSL. Literal: \"Why it's beneficial to try Linux now\"", - "body": "
Direct url
", - "author": "noname7890", - "publication_date": "2020-07-19T15:19:27Z", - "url": "https://www.reddit.com/r/linux/comments/hu0d5v/german_article_in_major_german_newspaper_about/", - "read": false, - "rule": 80, - "remote_identifier": "hu0d5v" - } -}, -{ - "model": "core.post", - "pk": 3117, - "fields": { - "created": "2020-07-20T19:32:35.837Z", - "modified": "2020-07-21T20:14:50.574Z", - "title": "Brian Kernighan: UNIX, C, AWK, AMPL, and Go Programming | AI Podcast #109 with Lex Fridman", - "body": "
Direct url
", - "author": "tinyatom", - "publication_date": "2020-07-20T08:48:35Z", - "url": "https://www.reddit.com/r/linux/comments/hugn0w/brian_kernighan_unix_c_awk_ampl_and_go/", - "read": false, - "rule": 80, - "remote_identifier": "hugn0w" - } -}, -{ - "model": "core.post", - "pk": 3118, - "fields": { - "created": "2020-07-20T19:32:35.837Z", - "modified": "2020-07-21T20:14:50.578Z", - "title": "Explaining Computers Host Christopher Barnatt Has Switched To Linux", - "body": "
Direct url
", - "author": "sysrpl", - "publication_date": "2020-07-20T13:00:02Z", - "url": "https://www.reddit.com/r/linux/comments/hujb12/explaining_computers_host_christopher_barnatt_has/", - "read": false, - "rule": 80, - "remote_identifier": "hujb12" - } -}, -{ - "model": "core.post", - "pk": 3119, - "fields": { - "created": "2020-07-20T19:32:35.837Z", - "modified": "2020-07-21T20:14:50.529Z", - "title": "Ireland donates contact tracing app to the Linux foundation.", - "body": "
Direct url
", - "author": "mathiasryan", - "publication_date": "2020-07-20T21:31:43Z", - "url": "https://www.reddit.com/r/linux/comments/hury4e/ireland_donates_contact_tracing_app_to_the_linux/", - "read": false, - "rule": 80, - "remote_identifier": "hury4e" - } -}, -{ - "model": "core.post", - "pk": 3120, - "fields": { - "created": "2020-07-20T19:32:35.842Z", - "modified": "2020-07-21T20:14:50.588Z", - "title": "I implemented a simple terminal-based password manager", - "body": "

I created a simple, secure, and free password manager written in C: SaltPass. I haven't contributed open source code before, but I think this might be useful to a few people. Especially as an alternative to paid solutions such as LastPass and the likes. Any suggestions/edits/code improvements would be greatly appreciated!

\n
", - "author": "zaid-gg", - "publication_date": "2020-07-20T07:43:03Z", - "url": "https://www.reddit.com/r/linux/comments/hufula/i_implemented_a_simple_terminalbased_password/", - "read": false, - "rule": 80, - "remote_identifier": "hufula" - } -}, -{ - "model": "core.post", - "pk": 3121, - "fields": { - "created": "2020-07-20T19:32:35.843Z", - "modified": "2020-07-21T20:14:50.593Z", - "title": "Performance analysis of multi services on container Docker, LXC, and LXD - Bulletin of Electrical Engineering and Informatics, Adinda Riztia Putri, Rendy Munadi, Ridha Muldina Negara Adaptive Network\u2026", - "body": "
Direct url
", - "author": "bmullan", - "publication_date": "2020-07-20T11:35:59Z", - "url": "https://www.reddit.com/r/linux/comments/huieio/performance_analysis_of_multi_services_on/", - "read": false, - "rule": 80, - "remote_identifier": "huieio" - } -}, -{ - "model": "core.post", - "pk": 3122, - "fields": { - "created": "2020-07-20T19:32:35.844Z", - "modified": "2020-07-21T20:14:50.602Z", - "title": "Create an Internal PKI using OpenSSL and NitroKey HSM", - "body": "
Direct url
", - "author": "PixelPaulaus", - "publication_date": "2020-07-20T06:18:41Z", - "url": "https://www.reddit.com/r/linux/comments/huerpn/create_an_internal_pki_using_openssl_and_nitrokey/", - "read": false, - "rule": 80, - "remote_identifier": "huerpn" - } -}, -{ - "model": "core.post", - "pk": 3123, - "fields": { - "created": "2020-07-20T19:32:35.844Z", - "modified": "2020-07-20T19:32:35.883Z", - "title": "vopono - run applications via VPNs with temporary network namespaces", - "body": "
Direct url
", - "author": "nivenkos", - "publication_date": "2020-07-19T20:02:57Z", - "url": "https://www.reddit.com/r/linux/comments/hu4vge/vopono_run_applications_via_vpns_with_temporary/", - "read": false, - "rule": 80, - "remote_identifier": "hu4vge" - } -}, -{ - "model": "core.post", - "pk": 3124, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.886Z", - "title": "Double (triple, quadruple...) internet speed with openvpn tap channel bonding to a linux VPS", - "body": "

I have been working a couple of days on my latest video about channel bonding - the video is heavily inspired be this article on Serverfault. In essence, I have been searching for a while on how to bond multiple VPN channels together in order to increase internet speed - there does not seem to be a lot of information around - mainly articles on forums and reddit state that it should be possible but a detailed guide is hard to find. I am using two Ubuntu machines in order to build the connection - one local and one VPS. The bash scripts I use in my video in order to achieve tap channel bonding are available on my github repository. I am currently working on a second video in order to walk through and explain the scripts in depth. Enjoy!

\n\n

(EDIT) - the question has come up in the discussions below if this is really packet load balancing or rather balancing links only - please see my comment further down - I can confirm that this DOES packet balancing so it does work as described.

\n
", - "author": "onemarcfifty", - "publication_date": "2020-07-19T20:41:40Z", - "url": "https://www.reddit.com/r/linux/comments/hu5l4f/double_triple_quadruple_internet_speed_with/", - "read": false, - "rule": 80, - "remote_identifier": "hu5l4f" - } -}, -{ - "model": "core.post", - "pk": 3125, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.888Z", - "title": "OpenRGB - Open source RGB lighting control that doesn't depend on manufacturer software, supports Linux", - "body": "
Direct url
", - "author": "pr0_c0d3", - "publication_date": "2020-07-18T16:52:48Z", - "url": "https://www.reddit.com/r/linux/comments/hthuli/openrgb_open_source_rgb_lighting_control_that/", - "read": false, - "rule": 80, - "remote_identifier": "hthuli" - } -}, -{ - "model": "core.post", - "pk": 3126, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.890Z", - "title": "Make this any sense? Automatic CPU Speed & Power Optimizer", - "body": "
Direct url
", - "author": "spite77", - "publication_date": "2020-07-20T11:53:35Z", - "url": "https://www.reddit.com/r/linux/comments/huikxz/make_this_any_sense_automatic_cpu_speed_power/", - "read": false, - "rule": 80, - "remote_identifier": "huikxz" - } -}, -{ - "model": "core.post", - "pk": 3127, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.891Z", - "title": "Let\u2019s not be pedantic about \u201cOpen Source\u201d", - "body": "
Direct url
", - "author": "speckz", - "publication_date": "2020-07-20T16:46:43Z", - "url": "https://www.reddit.com/r/linux/comments/humirw/lets_not_be_pedantic_about_open_source/", - "read": false, - "rule": 80, - "remote_identifier": "humirw" - } -}, -{ - "model": "core.post", - "pk": 3128, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.893Z", - "title": "Experiences with running Linux Lite", - "body": "
Direct url
", - "author": "daemonpenguin", - "publication_date": "2020-07-20T02:43:49Z", - "url": "https://www.reddit.com/r/linux/comments/hubonw/experiences_with_running_linux_lite/", - "read": false, - "rule": 80, - "remote_identifier": "hubonw" - } -}, -{ - "model": "core.post", - "pk": 3129, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.895Z", - "title": "Tried gnome on arch, surprised how lean it is (used flameshot so it used about 72mb more) closing at 600 megs) on fedora and pop i had gnome eating up 1.3gigs at boot up.", - "body": "
\"Tried
", - "author": "V1n0dKr1shna", - "publication_date": "2020-07-18T13:54:55Z", - "url": "https://www.reddit.com/r/linux/comments/htfeph/tried_gnome_on_arch_surprised_how_lean_it_is_used/", - "read": false, - "rule": 80, - "remote_identifier": "htfeph" - } -}, -{ - "model": "core.post", - "pk": 3130, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.897Z", - "title": "The Free Software Foundation is holding a Fundraiser, help them reach 200 members", - "body": "
Direct url
", - "author": "Neet-Feet", - "publication_date": "2020-07-18T17:55:30Z", - "url": "https://www.reddit.com/r/linux/comments/htiuyi/the_free_software_foundation_is_holding_a/", - "read": false, - "rule": 80, - "remote_identifier": "htiuyi" - } -}, -{ - "model": "core.post", - "pk": 3131, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.899Z", - "title": "Why is the mindset around Arch so negative?", - "body": "

I love the Linux community as a whole. You can find some of the most creative and imaginative people within most Linux communities. On a whole, Linux users are some of the most helpful and informative people you can encounter. Truly the type to think outside the box and learn new things. It can be very inspirational.

\n\n

If I jumped onto Ubuntu, Fedora, or openSUSE's community I can have a free flowing conversation about Linux, their distribution, and getting help or giving help is so free-flowing and easy. The communities are eager to welcome new people and appreciate folks who contribute.

\n\n

Then you have Arch. I love the OS but dislike the mindset. Asking for help is meat with resistance, giving help can also be punishable, and god forbid you try to have a discussion. But it's not just their core community either. For example, I just discovered Endeavour OS which is built around Arch and after 11 post I'm told to come back in 8 hours. Their subReddit here on Reddit, you have to ask to even make 1 post. There of course is also Manjaro Linux and they too have this gatekeeper mindset, the same can be said for ArcoLinux.

\n\n

What is it about Arch that makes everyone want to be either a control freak or a gatekeeper?

\n\n

I do not see this within the Ubuntu or Fedora or openSUSE communities. As I said, their mindset seems eager and willing to unite and work as a community. Am I the only how has noticed this?

\n
", - "author": "Linux-Is-Best", - "publication_date": "2020-07-18T23:28:12Z", - "url": "https://www.reddit.com/r/linux/comments/htojwk/why_is_the_mindset_around_arch_so_negative/", - "read": false, - "rule": 80, - "remote_identifier": "htojwk" - } -}, -{ - "model": "core.post", - "pk": 3132, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.901Z", - "title": "Using the nstat network statistics command in Linux", - "body": "
Direct url
", - "author": "cronos426", - "publication_date": "2020-07-19T17:55:55Z", - "url": "https://www.reddit.com/r/linux/comments/hu2q6v/using_the_nstat_network_statistics_command_in/", - "read": false, - "rule": 80, - "remote_identifier": "hu2q6v" - } -}, -{ - "model": "core.post", - "pk": 3133, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.903Z", - "title": "Contributing via GitLab Merge Requests", - "body": "
Direct url
", - "author": "ChristophCullmann", - "publication_date": "2020-07-18T20:01:26Z", - "url": "https://www.reddit.com/r/linux/comments/htl05p/contributing_via_gitlab_merge_requests/", - "read": false, - "rule": 80, - "remote_identifier": "htl05p" - } -}, -{ - "model": "core.post", - "pk": 3134, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.905Z", - "title": "OpenMandriva: combines WINE64 and 32 into one package capable of running both binaries, i686 architecture was considered as deprecated. Work is underway on a new Rolling release", - "body": "
Direct url
", - "author": "DamonsLinux", - "publication_date": "2020-07-18T15:02:35Z", - "url": "https://www.reddit.com/r/linux/comments/htg9dj/openmandriva_combines_wine64_and_32_into_one/", - "read": false, - "rule": 80, - "remote_identifier": "htg9dj" - } -}, -{ - "model": "core.post", - "pk": 3135, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.906Z", - "title": "OpenRCT2 Player Survey 2020 - Previous survey shows almost 25% players are linux, please help represent linux in the most recent survey", - "body": "
Direct url
", - "author": "christophski", - "publication_date": "2020-07-18T11:39:06Z", - "url": "https://www.reddit.com/r/linux/comments/htdzuh/openrct2_player_survey_2020_previous_survey_shows/", - "read": false, - "rule": 80, - "remote_identifier": "htdzuh" - } -}, -{ - "model": "core.post", - "pk": 3136, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.908Z", - "title": "This week in KDE: Get New Stuff fixes and more", - "body": "
Direct url
", - "author": "kyentei", - "publication_date": "2020-07-18T10:03:46Z", - "url": "https://www.reddit.com/r/linux/comments/htd1an/this_week_in_kde_get_new_stuff_fixes_and_more/", - "read": false, - "rule": 80, - "remote_identifier": "htd1an" - } -}, -{ - "model": "core.post", - "pk": 3137, - "fields": { - "created": "2020-07-20T19:32:35.857Z", - "modified": "2020-07-20T19:32:35.910Z", - "title": "Blender Runs on Linux Pinephone", - "body": "

I managed to get the desktop version of Blender on the Pinephone, and it works really well except for a few bugs.

\n\n

See my post on r/blender:

\n\n

https://www.reddit.com/r/blender/comments/hsxv27/i_installed_blender_on_a_phone/

\n\n

and r/PINE64official:

\n\n

https://www.reddit.com/r/PINE64official/comments/hsxc33/blender_on_pine_phone_almost_usable/

\n\n

I've tried other desktop programs like Xournal and PPSSPP, their UIs also work well, I'd be able to do even more if OpenGL 3 was working.

\n
", - "author": "InfiniteHawk", - "publication_date": "2020-07-17T22:35:14Z", - "url": "https://www.reddit.com/r/linux/comments/ht3d4k/blender_runs_on_linux_pinephone/", - "read": false, - "rule": 80, - "remote_identifier": "ht3d4k" - } -}, -{ - "model": "core.post", - "pk": 3138, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:18:21.616Z", - "title": "Hrmmm They Need to Fix Throttle Animations in the Sabre", - "body": "
", - "author": "TheBootRanger", - "publication_date": "2020-07-21T13:26:01Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv5omc/hrmmm_they_need_to_fix_throttle_animations_in_the/", - "read": true, - "rule": 82, - "remote_identifier": "hv5omc" - } -}, -{ - "model": "core.post", - "pk": 3139, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:18:49.999Z", - "title": "My first 3.10 landing could have gone better...", - "body": "
", - "author": "KnLfey", - "publication_date": "2020-07-21T16:04:50Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv7w85/my_first_310_landing_could_have_gone_better/", - "read": true, - "rule": 82, - "remote_identifier": "hv7w85" - } -}, -{ - "model": "core.post", - "pk": 3140, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:14:50.439Z", - "title": "How about the Christmas in 3 more years?", - "body": "
\"How
", - "author": "SpleanEater", - "publication_date": "2020-07-21T17:49:22Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv9qy8/how_about_the_christmas_in_3_more_years/", - "read": false, - "rule": 82, - "remote_identifier": "hv9qy8" - } -}, -{ - "model": "core.post", - "pk": 3141, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:18:33.532Z", - "title": "Long time Elite Dangerous player. New to star citizen i think im doing great", - "body": "
Direct url
", - "author": "Filblo5", - "publication_date": "2020-07-21T15:33:49Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv7elb/long_time_elite_dangerous_player_new_to_star/", - "read": true, - "rule": 82, - "remote_identifier": "hv7elb" - } -}, -{ - "model": "core.post", - "pk": 3142, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.443Z", - "title": "And we stand by it.", - "body": "
\"And
", - "author": "CyberTill", - "publication_date": "2020-07-21T18:57:48Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvb3wm/and_we_stand_by_it/", - "read": false, - "rule": 82, - "remote_identifier": "hvb3wm" - } -}, -{ - "model": "core.post", - "pk": 3143, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.446Z", - "title": "Nomad", - "body": "
\"Nomad\"
", - "author": "ibracitizen", - "publication_date": "2020-07-21T19:52:24Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvc5h3/nomad/", - "read": false, - "rule": 82, - "remote_identifier": "hvc5h3" - } -}, -{ - "model": "core.post", - "pk": 3144, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.449Z", - "title": "Probably the best screen cap i've ever caught on a whim. 3.5 Arc Corp release. Also a confession: I never pledged. Got a ship with my GPU. I intend to pay my dues.", - "body": "
\"Probably
", - "author": "ScionoicS", - "publication_date": "2020-07-21T20:23:01Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvcqzf/probably_the_best_screen_cap_ive_ever_caught_on_a/", - "read": false, - "rule": 82, - "remote_identifier": "hvcqzf" - } -}, -{ - "model": "core.post", - "pk": 3145, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.451Z", - "title": "Play to escape the depressing job hunt where I need 10 years experience for a entry level job to find this, only been playing for 1 and a half years :(", - "body": "
\"Play
", - "author": "Albert-III-", - "publication_date": "2020-07-21T12:23:45Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv4z08/play_to_escape_the_depressing_job_hunt_where_i/", - "read": false, - "rule": 82, - "remote_identifier": "hv4z08" - } -}, -{ - "model": "core.post", - "pk": 3146, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:19:00.691Z", - "title": "The void beckons.", - "body": "
", - "author": "HisNameWasHis", - "publication_date": "2020-07-21T14:40:51Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv6nij/the_void_beckons/", - "read": true, - "rule": 82, - "remote_identifier": "hv6nij" - } -}, -{ - "model": "core.post", - "pk": 3147, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:19:05.881Z", - "title": "I made a SC-like Photobash with Soldiers", - "body": "
\"I
", - "author": "IsaacPolar", - "publication_date": "2020-07-21T17:13:39Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv92ri/i_made_a_sclike_photobash_with_soldiers/", - "read": true, - "rule": 82, - "remote_identifier": "hv92ri" - } -}, -{ - "model": "core.post", - "pk": 3148, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:19:41.227Z", - "title": "Ocean Shader Improvements", - "body": "
\"Ocean
", - "author": "shoeii", - "publication_date": "2020-07-21T18:41:51Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvasds/ocean_shader_improvements/", - "read": true, - "rule": 82, - "remote_identifier": "hvasds" - } -}, -{ - "model": "core.post", - "pk": 3149, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.459Z", - "title": "As much shit as Star Citizen (rightfully) gets it still does one thing better than any other 'game' I've played", - "body": "

It invokes a real sense of scale, on multiple levels.

\n\n

One could argue that's one of the most important feelings you'd want to capture in any game set in space, but of course it's mostly meaningless if there aren't enough gameplay loops and systems in place to work in tandem with and make the space that's been created interesting, and that's where SC is currently a failure.

\n\n

Even so, I think being able to create that sense of smallness isn't insignificant.

\n\n

You as a pilot are dwarfed by your ship which is itself dwarfed by a larger ship which is itself dwarfed by another, even more massive one which is dwarfed by the space station or hub you're at which is dwarfed by a crater on a moon which is dwarfed by the moon itself which is dwarfed by the planet it orbits which is dwarfed by the sheer vastness of space in between all of those things and that they are, despite the distance, still connected.

\n\n

Getting lost in Lorville (even if it is mostly linear) and knowing it's only a small part of the playable space is a really neat feeling - looking out from the windows of the train up into the sky and knowing you can go there and beyond really makes you feel like there is a whole world (and more) waiting to be explored.

\n\n

I think this is a direct result of having legs and not being locked into the cockpit of your ship - I've played more Elite: Dangerous than Star Citizen and it accomplishes a similar sense of scale but, at least not as far as I've felt, never to the same degree - because you're locked in your ship you never really get this same sense of being small or insignificant even though you are dwarfed in similar ways by planets/asteroids/other ships - will be interesting to see how their implementation of 'space legs' in the upcoming expansion changes this.

\n\n

My favourite thing to do in Star Citizen (because there isn't a whole lot) is to just find some pocket of space far away from anything else and just walk around my ship, feeling truly alone and insignificant, gazing out at the void that stretches infinitely all around - something about that is super comfy.

\n\n

I can't think of many other game that accomplish a similar level of scale though I'm sure they exist.

\n\n

I've been playing an indie game called Empyrion - Galactic Survival and it actually is sort of similar to SC in this regard but it's nowhere near as polished or smooth - transitions from atmosphere to space are not truly seamless and planets themselves are kind of stitched together, but it still manages to invoke that same kind of awe at the scale of things when you dock a small vessel to a capital vessel, for example - definitely worth checking out if you like sci-fi/space games, which you must if you're here, but just be prepared for the jank.

\n
", - "author": "thegreatself", - "publication_date": "2020-07-21T20:30:15Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvcw38/as_much_shit_as_star_citizen_rightfully_gets_it/", - "read": false, - "rule": 82, - "remote_identifier": "hvcw38" - } -}, -{ - "model": "core.post", - "pk": 3150, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.462Z", - "title": "You waiting for patch 3.10 to go live while watching tons of videos about the new flight model features. Be patient, 3.11 and 3.12 will be even better.", - "body": "
\"You
", - "author": "jsabater76", - "publication_date": "2020-07-21T09:39:27Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv372v/you_waiting_for_patch_310_to_go_live_while/", - "read": false, - "rule": 82, - "remote_identifier": "hv372v" - } -}, -{ - "model": "core.post", - "pk": 3151, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.466Z", - "title": "CIG, can we please fix these \"black hole\" doors(when they are closed) on ships please.", - "body": "
\"CIG,
", - "author": "AbnormallyBendPenis", - "publication_date": "2020-07-21T13:40:14Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv5uzj/cig_can_we_please_fix_these_black_hole_doorswhen/", - "read": false, - "rule": 82, - "remote_identifier": "hv5uzj" - } -}, -{ - "model": "core.post", - "pk": 3152, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.468Z", - "title": "Anvil Super Hornet over Cellin", - "body": "
\"Anvil
", - "author": "SaraCaterina", - "publication_date": "2020-07-21T20:33:58Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvcyq6/anvil_super_hornet_over_cellin/", - "read": false, - "rule": 82, - "remote_identifier": "hvcyq6" - } -}, -{ - "model": "core.post", - "pk": 3153, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.471Z", - "title": "3.10 Combat Changes", - "body": "
Direct url
", - "author": "STLYoungblood", - "publication_date": "2020-07-21T16:37:44Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv8fr7/310_combat_changes/", - "read": false, - "rule": 82, - "remote_identifier": "hv8fr7" - } -}, -{ - "model": "core.post", - "pk": 3154, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.472Z", - "title": "Hey CIG how about that S42 Vi.... Oh...", - "body": "
\"Hey
", - "author": "SiEDeN", - "publication_date": "2020-07-21T21:37:16Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hve6am/hey_cig_how_about_that_s42_vi_oh/", - "read": false, - "rule": 82, - "remote_identifier": "hve6am" - } -}, -{ - "model": "core.post", - "pk": 3155, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.475Z", - "title": "3.10 M PTU Eclipse improvements", - "body": "

If this goes live, CIG had addressed 2 of my Eclipse critics.

\n\n

Not because of my videos of course, CIG doesn't know I exist.

\n\n

 

\n\n

a. Eclipse has armor stealth in 3.10, see my table:\nhttps://docs.google.com/spreadsheets/d/1OJXg7MQsG_IVTPsmlmZYaxEPK4n4iqnhQx4oigIlJHg/edit#gid=343807746

\n\n

 

\n\n

b. Eclipse can fire her size 9 torpedoes way quicker now, see my video with a side by side comparison of the max firing speed in 3.9 and 3.10:\nhttps://youtu.be/GFTF1Qt7T3o?t=207

\n
", - "author": "Camural", - "publication_date": "2020-07-21T18:15:50Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hva9lc/310_m_ptu_eclipse_improvements/", - "read": false, - "rule": 82, - "remote_identifier": "hva9lc" - } -}, -{ - "model": "core.post", - "pk": 3156, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.477Z", - "title": "Hark! The Drake Herald Sings", - "body": "
\"Hark!
", - "author": "CyrexStorm", - "publication_date": "2020-07-21T16:19:31Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv84kk/hark_the_drake_herald_sings/", - "read": false, - "rule": 82, - "remote_identifier": "hv84kk" - } -}, -{ - "model": "core.post", - "pk": 3157, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.479Z", - "title": "The new flight stick in the Prowler", - "body": "
\"The
", - "author": "Potato_Nades", - "publication_date": "2020-07-21T16:22:22Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv86c2/the_new_flight_stick_in_the_prowler/", - "read": false, - "rule": 82, - "remote_identifier": "hv86c2" - } -}, -{ - "model": "core.post", - "pk": 3158, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.481Z", - "title": "Norwegian VAT charged from August 1st", - "body": "
\"Norwegian
", - "author": "norgeek", - "publication_date": "2020-07-21T10:30:57Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv3r3l/norwegian_vat_charged_from_august_1st/", - "read": false, - "rule": 82, - "remote_identifier": "hv3r3l" - } -}, -{ - "model": "core.post", - "pk": 3159, - "fields": { - "created": "2020-07-21T20:14:50.423Z", - "modified": "2020-07-21T20:14:50.484Z", - "title": "With Pyro (currently WIP), Nyx (partially done), Odin (S42), currently on the way, what is everyone\u2019s thoughts on Terra possibly being next on the list of star systems to be added into the PU within \u2026", - "body": "
\"With
", - "author": "realCLTotaku", - "publication_date": "2020-07-21T13:27:09Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv5p41/with_pyro_currently_wip_nyx_partially_done_odin/", - "read": false, - "rule": 82, - "remote_identifier": "hv5p41" - } -}, -{ - "model": "core.post", - "pk": 3160, - "fields": { - "created": "2020-07-21T20:14:50.423Z", - "modified": "2020-07-21T20:14:50.486Z", - "title": "Testing out the new electron rifle", - "body": "
", - "author": "joshbaker2112", - "publication_date": "2020-07-21T02:56:19Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huxr6d/testing_out_the_new_electron_rifle/", - "read": false, - "rule": 82, - "remote_identifier": "huxr6d" - } -}, -{ - "model": "core.post", - "pk": 3161, - "fields": { - "created": "2020-07-21T20:14:50.423Z", - "modified": "2020-07-21T20:14:50.487Z", - "title": "Imperial Geographic's Lovecraftian magazine special is here. \ud83d\udc19 Find the link in the comments!", - "body": "
\"Imperial
", - "author": "Good_Punk2", - "publication_date": "2020-07-21T18:21:38Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvadrh/imperial_geographics_lovecraftian_magazine/", - "read": false, - "rule": 82, - "remote_identifier": "hvadrh" - } -}, -{ - "model": "core.post", - "pk": 3162, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.525Z", - "title": "Linux Distributions Timeline", - "body": "
\"Linux
", - "author": "bauripalash", - "publication_date": "2020-07-21T06:07:59Z", - "url": "https://www.reddit.com/r/linux/comments/hv0ktn/linux_distributions_timeline/", - "read": false, - "rule": 80, - "remote_identifier": "hv0ktn" - } -}, -{ - "model": "core.post", - "pk": 3163, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.527Z", - "title": "Fedora: Proposal to replace default wined3d backend with DXVK", - "body": "
Direct url
", - "author": "friskfrugt", - "publication_date": "2020-07-21T19:42:49Z", - "url": "https://www.reddit.com/r/linux/comments/hvbyyr/fedora_proposal_to_replace_default_wined3d/", - "read": false, - "rule": 80, - "remote_identifier": "hvbyyr" - } -}, -{ - "model": "core.post", - "pk": 3164, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.531Z", - "title": "Update on marketing and communication plans for the LibreOffice 7.x series", - "body": "
Direct url
", - "author": "TheQuantumZero", - "publication_date": "2020-07-21T09:59:23Z", - "url": "https://www.reddit.com/r/linux/comments/hv3erm/update_on_marketing_and_communication_plans_for/", - "read": false, - "rule": 80, - "remote_identifier": "hv3erm" - } -}, -{ - "model": "core.post", - "pk": 3165, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.533Z", - "title": "FOSS job opening: LibreOffice Development Mentor at The Document Foundation", - "body": "
Direct url
", - "author": "themikeosguy", - "publication_date": "2020-07-21T14:26:36Z", - "url": "https://www.reddit.com/r/linux/comments/hv6gfw/foss_job_opening_libreoffice_development_mentor/", - "read": false, - "rule": 80, - "remote_identifier": "hv6gfw" - } -}, -{ - "model": "core.post", - "pk": 3166, - "fields": { - "created": "2020-07-21T20:14:50.503Z", - "modified": "2020-07-21T20:14:50.536Z", - "title": "gomd - quickly display formatted markdown files with code highlight in your browser", - "body": "

Hi all!

\n\n

I wanted to share a project I've been working on recently. I think it reached a stage where it's pretty usable and should work out of the box. gomd sets up a HTTP server and serves a directory in your browser so you can quickly view your markdown files. It comes with some neat features like:

\n\n\n\n

Link: gomd

\n\n

For now its only available from AUR or built from source.

\n\n

\n\n

Any tips or feedback will be greatly appreciated :)

\n
", - "author": "wwojtekk", - "publication_date": "2020-07-21T20:07:31Z", - "url": "https://www.reddit.com/r/linux/comments/hvcg44/gomd_quickly_display_formatted_markdown_files/", - "read": false, - "rule": 80, - "remote_identifier": "hvcg44" - } -}, -{ - "model": "core.post", - "pk": 3167, - "fields": { - "created": "2020-07-21T20:14:50.503Z", - "modified": "2020-07-21T20:14:50.543Z", - "title": "They're not otherwise wrong, but it didn't become a real Internet standard until 2017.", - "body": "
\"They're
", - "author": "foodown", - "publication_date": "2020-07-21T21:39:09Z", - "url": "https://www.reddit.com/r/linux/comments/hve7l5/theyre_not_otherwise_wrong_but_it_didnt_become_a/", - "read": false, - "rule": 80, - "remote_identifier": "hve7l5" - } -}, -{ - "model": "core.post", - "pk": 3168, - "fields": { - "created": "2020-07-21T20:14:50.503Z", - "modified": "2020-07-21T20:14:50.545Z", - "title": "Drawing - an alternative to Paint for Linux (gtk3, support HiDPI)", - "body": "
Direct url
", - "author": "dontdieych", - "publication_date": "2020-07-21T02:37:22Z", - "url": "https://www.reddit.com/r/linux/comments/huxgsg/drawing_an_alternative_to_paint_for_linux_gtk3/", - "read": false, - "rule": 80, - "remote_identifier": "huxgsg" - } -}, -{ - "model": "core.post", - "pk": 3169, - "fields": { - "created": "2020-07-21T20:14:50.509Z", - "modified": "2020-07-21T20:14:50.547Z", - "title": "Observations on a Linux issue with 3.5mm earphones with a mic", - "body": "

Alright hello. I have come from r/SolusProject and I made a post there to do with headphone issues. I suggest you read through the post and comments to get a better understanding before reading this https://www.reddit.com/r/SolusProject/comments/hsql4d/frustrating_headphone_issues/. I had posted to do with it again, but it got taken down for duplication (when it wasn't duplication). This post is more of my observations from experimenting and such. There are distros I haven't tried but I tried a wide range of distros like manjaro, ubuntu based ones and all solus flavors, and I was looking more for how well they worked out of the box, rather than with fiddling around with pulse, hdajack etc which I know will work eventually. If you stumbled across this from searching about the same issue I have (or similar) or are confused to what this is about, I suggest you look at my previous post also.

\n\n

So anyways, I've tried the past few days mounting isos to usb drives and trying live os and installing various distros to see about the headphone issue. And my conclusion is that this issue affects the linux kernel in some way across the board. I don't really understand why completely but I have some kind of idea.

\n\n

From installing fresh distros, I noticed that the earphones (they are 3.5mm earphones + mic) get recognised as a microphone and not as a speaker system of some kind. Every single time I had a look at the sound settings and in pulse, they came up as plugged microphone, with the internal speakers being the only output device every single time. It's really odd seeing as how ubuntu 14.04 and xubuntu etc from years past worked flawlessly with the earphones, even manjaro a while ago on my older craptop worked fine. I don't really understand why it doesn't work on my device now.

\n\n

I'll leave my specs at the bottom of this post but what I think is is there's something the manufacturer did, or something like the cpu causes issue with linux. The manufacturer of my laptop is Lenovo, and the cpu/igpu is from AMD. A warning sign is that when installing a linux distro, it doesn't bring up the dual boot menu at startup like it should. Instead it completely hides the fact it exists until I use something like easyuefi to add an option for that distro, how it works is you specify the boot partition, whether it's linux or windows and the loader conf file for the distro. All of this hassle everytime doesn't appear on my craptop, because the dual boot menu appears flawlessly without issue. May be because it uses an Intel cpu/igpu unlike my newer laptop but it's hard to say.

\n\n

Also, it seems like the devices that appear in a given distro when looking at alsa, is hd generic devices but by reloading alsa or any command that shows the full name of the device, it says it's Intel. I don't know if that would be an issue, maybe amd use intel sound drivers or something. It's odd nonetheless.

\n\n

This issue has been boggling my mind for obvious reasons, with half-rhetorical questions like does linux not support the earphones anymore, whether out of accident from an overlooked bug in an update or intentionally phasing out? Is any of this AMD or Lenovo's fault? Even with proper headphones or something, will they fail? I don't think anyone here really knows, hell I'd bet an extreme that no one really understands why in the linux community. I kinda rambled in this post with stuff that should've been said in the last post/thread, but I'm saying it now.

\n\n

Thanks for contributing thus far to this discussion in figuring this out.

\n\n

Specs: AMD Ryzen 5 3500U Mobile CPU (2.2 - 3.7ghz quad core)

\n\n

Radeon Vega 8 Integrated GPU, 8GB Ram, 256GB SSD.

\n\n

Lenovo C340-14API Laptop

\n
", - "author": "BrianMeerkatlol", - "publication_date": "2020-07-21T21:02:19Z", - "url": "https://www.reddit.com/r/linux/comments/hvdi3o/observations_on_a_linux_issue_with_35mm_earphones/", - "read": false, - "rule": 80, - "remote_identifier": "hvdi3o" - } -}, -{ - "model": "core.post", - "pk": 3170, - "fields": { - "created": "2020-07-21T20:14:50.509Z", - "modified": "2020-07-21T20:14:50.549Z", - "title": "South Korean distro HamoniKR OS has been added to Distrowatch", - "body": "
Direct url
", - "author": "TheHordeRisesAgain", - "publication_date": "2020-07-21T07:44:21Z", - "url": "https://www.reddit.com/r/linux/comments/hv1ug1/south_korean_distro_hamonikr_os_has_been_added_to/", - "read": false, - "rule": 80, - "remote_identifier": "hv1ug1" - } -}, -{ - "model": "core.post", - "pk": 3171, - "fields": { - "created": "2020-07-21T20:14:50.509Z", - "modified": "2020-07-21T20:14:50.559Z", - "title": "The Jailer is free! New release of the outstanding database subsetter and browser is available.", - "body": "
Direct url
", - "author": "Plane-Discussion", - "publication_date": "2020-07-21T12:53:54Z", - "url": "https://www.reddit.com/r/linux/comments/hv5b0j/the_jailer_is_free_new_release_of_the_outstanding/", - "read": false, - "rule": 80, - "remote_identifier": "hv5b0j" - } -}, -{ - "model": "core.post", - "pk": 3172, - "fields": { - "created": "2020-07-21T20:14:50.513Z", - "modified": "2020-07-21T20:14:50.563Z", - "title": "A few very well-aged excerpts from Microsoft\u2019s infamous 2004 \u201cGet the facts\u201d campaign, where they make the case for Windows servers being cheaper, more secure, and more performant than Linux servers", - "body": "
\n

Get the facts on Windows and Linux.

\n\n

Leading companies and third-party analysts confirm it: Windows has a lower total cost of ownership and outperforms Linux.

\n\n

...

\n\n

-Security

\n\n

Windows Users Have Fewer Vulnerabilities

\n
\n\n

And then literally the very next bullet point:

\n\n
\n

-Featured Customer Case Study

\n\n

Equifax

\n\n

Equifax Sees 14 Percent Cost Savings

\n\n

Find out why Equifax, a global leader in transforming data into intelligence, selected Windows over Linux to enhance the speed and performance of its marketing services capabilities. Using Microsoft Windows Server System, the company has seen 14 percent in cost savings over Linux.

\n
\n\n

Good thing they saved 14% and got all that extra security! Sure their website is janky and their login flow is downright horrifying (Check it out if you want to be amazed), but who could blame them? Linux is \u201cProhibitively Expensive, Extremely Complex, and Provides No Tangible Business Gains\u201d, Microsoft said so!

\n\n

Source: https://web.archive.org/web/20041027003759/http://www.microsoft.com/windowsserversystem/facts/default.mspx

\n
", - "author": "kevinhaze", - "publication_date": "2020-07-20T21:42:15Z", - "url": "https://www.reddit.com/r/linux/comments/hus5lz/a_few_very_wellaged_excerpts_from_microsofts/", - "read": false, - "rule": 80, - "remote_identifier": "hus5lz" - } -}, -{ - "model": "core.post", - "pk": 3173, - "fields": { - "created": "2020-07-21T20:14:50.515Z", - "modified": "2020-07-21T20:14:50.566Z", - "title": "Are there are any professional audio recording studios or artists that use Linux?", - "body": "

As the title says, who is using Linux as a professional audio engineer, producer, or artist? I am a former Mac user myself, and I am seeing people from time to time who have become disillusioned with what Apple has been doing for the past few years. However, I'm not sure if Linux really has a place for these people to land if they are serious about what they do.

\n\n

Fedora Design Suite and Ubuntu Studio are definitely encouraging to see, but what is their real-world usage like? Are we getting better with professional audio in Linux, or have things been stagnant for years?

\n
", - "author": "RootHouston", - "publication_date": "2020-07-21T00:08:26Z", - "url": "https://www.reddit.com/r/linux/comments/huuxvq/are_there_are_any_professional_audio_recording/", - "read": false, - "rule": 80, - "remote_identifier": "huuxvq" - } -}, -{ - "model": "core.post", - "pk": 3174, - "fields": { - "created": "2020-07-21T20:14:50.515Z", - "modified": "2020-07-21T20:14:50.570Z", - "title": "When Linux had marketing", - "body": "
Direct url
", - "author": "Commodore256", - "publication_date": "2020-07-21T14:03:56Z", - "url": "https://www.reddit.com/r/linux/comments/hv65oa/when_linux_had_marketing/", - "read": false, - "rule": 80, - "remote_identifier": "hv65oa" - } -}, -{ - "model": "core.post", - "pk": 3175, - "fields": { - "created": "2020-07-21T20:14:50.520Z", - "modified": "2020-07-21T20:14:50.598Z", - "title": "Ward: Simple and minimalistic server dashboard", - "body": "

Ward is a simple and and minimalistic server monitoring tool. Ward supports adaptive design system. Also it supports dark theme. It shows only principal information and can be used, if you want to see nice looking dashboard instead looking on bunch of numbers and graphs. Ward works nice on all popular operating systems, because it uses OSHI.

\n\n

https://preview.redd.it/gdppswc3a3c51.png?width=1448&format=png&auto=webp&s=0d6e10146c105ddcfd045dd59c970d4c127ddb8c

\n\n

https://github.com/B-Software/Ward

\n
", - "author": "Pabyzu", - "publication_date": "2020-07-21T00:33:40Z", - "url": "https://www.reddit.com/r/linux/comments/huvea3/ward_simple_and_minimalistic_server_dashboard/", - "read": false, - "rule": 80, - "remote_identifier": "huvea3" - } -}, -{ - "model": "core.post", - "pk": 3176, - "fields": { - "created": "2020-07-21T20:14:50.522Z", - "modified": "2020-07-21T20:14:50.606Z", - "title": "WindowsFX - a good Windows alternative?", - "body": "

I would personally like to hear some of your opinions (in the replies) about WindowsFX. What is WindowsFX you may ask? WindowsFX is a Brazilian linux distribution that is designed to look and act like Windows 10.

\n\n

Linux / WindowsFX is based off of Ubuntu, and uses Cinnamon as its DE. Upon first boot, normal Windows users can tell the difference. But if you were to put it in front of a non tech-savvy person, they wouldn't be able to tell the difference.

\n\n

Personally, with WSL on Windows, I see no need for a distro like this. However, as I said, I would like to hear your opinions on this distro.

\n\n

Video review here.

\n
", - "author": "Demonitized101", - "publication_date": "2020-07-20T23:03:29Z", - "url": "https://www.reddit.com/r/linux/comments/hutpt5/windowsfx_a_good_windows_alternative/", - "read": false, - "rule": 80, - "remote_identifier": "hutpt5" - } -}, -{ - "model": "core.post", - "pk": 3177, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:14:50.780Z", - "title": "Every day this good boy brings a carrot to his best buddy", - "body": "
", - "author": "TooShiftyForYou", - "publication_date": "2020-07-21T15:25:31Z", - "url": "https://www.reddit.com/r/aww/comments/hv7a8b/every_day_this_good_boy_brings_a_carrot_to_his/", - "read": false, - "rule": 81, - "remote_identifier": "hv7a8b" - } -}, -{ - "model": "core.post", - "pk": 3178, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-25T20:08:34.264Z", - "title": "Kitten mimics his human petting the dog", - "body": "
", - "author": "SpecterAscendant", - "publication_date": "2020-07-21T14:56:57Z", - "url": "https://www.reddit.com/r/aww/comments/hv6ve3/kitten_mimics_his_human_petting_the_dog/", - "read": true, - "rule": 81, - "remote_identifier": "hv6ve3" - } -}, -{ - "model": "core.post", - "pk": 3179, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:14:50.789Z", - "title": "My fox friend!", - "body": "
", - "author": "Zepantha", - "publication_date": "2020-07-21T14:27:25Z", - "url": "https://www.reddit.com/r/aww/comments/hv6gte/my_fox_friend/", - "read": false, - "rule": 81, - "remote_identifier": "hv6gte" - } -}, -{ - "model": "core.post", - "pk": 3180, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:15:46.876Z", - "title": "Ducks annihilate peas", - "body": "
", - "author": "tommycalibre", - "publication_date": "2020-07-21T17:12:40Z", - "url": "https://www.reddit.com/r/aww/comments/hv9258/ducks_annihilate_peas/", - "read": true, - "rule": 81, - "remote_identifier": "hv9258" - } -}, -{ - "model": "core.post", - "pk": 3181, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:14:50.797Z", - "title": "Wiggle it baby", - "body": "
", - "author": "neo_star", - "publication_date": "2020-07-21T18:44:31Z", - "url": "https://www.reddit.com/r/aww/comments/hvaucy/wiggle_it_baby/", - "read": false, - "rule": 81, - "remote_identifier": "hvaucy" - } -}, -{ - "model": "core.post", - "pk": 3182, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:16:22.725Z", - "title": "I guess I should do this.. everyone seems to be liking little pups and kittens so.. Reddit, meet bailey", - "body": "
\"I
", - "author": "X_XNOTHINGX_X", - "publication_date": "2020-07-21T14:15:08Z", - "url": "https://www.reddit.com/r/aww/comments/hv6b0a/i_guess_i_should_do_this_everyone_seems_to_be/", - "read": true, - "rule": 81, - "remote_identifier": "hv6b0a" - } -}, -{ - "model": "core.post", - "pk": 3183, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.806Z", - "title": "The hat makes the crab.", - "body": "
\"The
", - "author": "fujfuj", - "publication_date": "2020-07-21T14:48:40Z", - "url": "https://www.reddit.com/r/aww/comments/hv6rde/the_hat_makes_the_crab/", - "read": false, - "rule": 81, - "remote_identifier": "hv6rde" - } -}, -{ - "model": "core.post", - "pk": 3184, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.812Z", - "title": "Baby bunny fits in hand", - "body": "
", - "author": "Hawken10", - "publication_date": "2020-07-21T12:31:30Z", - "url": "https://www.reddit.com/r/aww/comments/hv5253/baby_bunny_fits_in_hand/", - "read": false, - "rule": 81, - "remote_identifier": "hv5253" - } -}, -{ - "model": "core.post", - "pk": 3185, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.818Z", - "title": "My cat and I, both pregnant", - "body": "
\"My
", - "author": "nixdionisio", - "publication_date": "2020-07-21T11:06:25Z", - "url": "https://www.reddit.com/r/aww/comments/hv44m2/my_cat_and_i_both_pregnant/", - "read": false, - "rule": 81, - "remote_identifier": "hv44m2" - } -}, -{ - "model": "core.post", - "pk": 3186, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.822Z", - "title": "Very sweet dance", - "body": "
", - "author": "Ashley1023", - "publication_date": "2020-07-21T13:03:03Z", - "url": "https://www.reddit.com/r/aww/comments/hv5ewq/very_sweet_dance/", - "read": false, - "rule": 81, - "remote_identifier": "hv5ewq" - } -}, -{ - "model": "core.post", - "pk": 3187, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.825Z", - "title": "My local pet-store has a cat named Vegemite \u2764\ufe0f", - "body": "
\"My
", - "author": "galinhad", - "publication_date": "2020-07-21T12:06:17Z", - "url": "https://www.reddit.com/r/aww/comments/hv4s5z/my_local_petstore_has_a_cat_named_vegemite/", - "read": false, - "rule": 81, - "remote_identifier": "hv4s5z" - } -}, -{ - "model": "core.post", - "pk": 3188, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-21T20:15:01.459Z", - "title": "A teacher like that makes a huge difference", - "body": "
", - "author": "Unicornglitteryblood", - "publication_date": "2020-07-21T18:29:57Z", - "url": "https://www.reddit.com/r/aww/comments/hvajo9/a_teacher_like_that_makes_a_huge_difference/", - "read": true, - "rule": 81, - "remote_identifier": "hvajo9" - } -}, -{ - "model": "core.post", - "pk": 3189, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-22T19:55:49.930Z", - "title": "Kitten Encounters Bubbly Water", - "body": "
\"Kitten
", - "author": "DragonOBunny", - "publication_date": "2020-07-21T15:28:05Z", - "url": "https://www.reddit.com/r/aww/comments/hv7bis/kitten_encounters_bubbly_water/", - "read": true, - "rule": 81, - "remote_identifier": "hv7bis" - } -}, -{ - "model": "core.post", - "pk": 3190, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-21T20:14:50.833Z", - "title": "Are These My Chickens Now?", - "body": "
Direct url
", - "author": "jasontaken", - "publication_date": "2020-07-21T09:55:36Z", - "url": "https://www.reddit.com/r/aww/comments/hv3de1/are_these_my_chickens_now/", - "read": false, - "rule": 81, - "remote_identifier": "hv3de1" - } -}, -{ - "model": "core.post", - "pk": 3191, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-25T20:08:20.518Z", - "title": "Our St Bernard 6 months apart", - "body": "
\"Our
", - "author": "ryan3105", - "publication_date": "2020-07-21T18:00:04Z", - "url": "https://www.reddit.com/r/aww/comments/hv9yea/our_st_bernard_6_months_apart/", - "read": true, - "rule": 81, - "remote_identifier": "hv9yea" - } -}, -{ - "model": "core.post", - "pk": 3192, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-21T20:14:50.837Z", - "title": "Father and child in sync", - "body": "
", - "author": "Araragi_Monogatari", - "publication_date": "2020-07-21T08:29:18Z", - "url": "https://www.reddit.com/r/aww/comments/hv2enj/father_and_child_in_sync/", - "read": false, - "rule": 81, - "remote_identifier": "hv2enj" - } -}, -{ - "model": "core.post", - "pk": 3193, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.840Z", - "title": "A meme is born", - "body": "
\"A
", - "author": "Unicornglitteryblood", - "publication_date": "2020-07-21T18:55:04Z", - "url": "https://www.reddit.com/r/aww/comments/hvb1vh/a_meme_is_born/", - "read": false, - "rule": 81, - "remote_identifier": "hvb1vh" - } -}, -{ - "model": "core.post", - "pk": 3194, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.842Z", - "title": "She bites, then she sleeps, then bites again, then sleeps again. \ud83d\ude02", - "body": "
", - "author": "earlymauvs", - "publication_date": "2020-07-21T11:34:19Z", - "url": "https://www.reddit.com/r/aww/comments/hv4fat/she_bites_then_she_sleeps_then_bites_again_then/", - "read": false, - "rule": 81, - "remote_identifier": "hv4fat" - } -}, -{ - "model": "core.post", - "pk": 3195, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.844Z", - "title": "Nothing calmer that 2 ginger cats rubbing heads and showing their love in morning", - "body": "
\"Nothing
", - "author": "Apotheosis33", - "publication_date": "2020-07-21T08:39:24Z", - "url": "https://www.reddit.com/r/aww/comments/hv2j2g/nothing_calmer_that_2_ginger_cats_rubbing_heads/", - "read": false, - "rule": 81, - "remote_identifier": "hv2j2g" - } -}, -{ - "model": "core.post", - "pk": 3196, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.851Z", - "title": "Ring Tailed Possum", - "body": "
Direct url
", - "author": "Wayward-Delver", - "publication_date": "2020-07-21T11:23:51Z", - "url": "https://www.reddit.com/r/aww/comments/hv4b9e/ring_tailed_possum/", - "read": false, - "rule": 81, - "remote_identifier": "hv4b9e" - } -}, -{ - "model": "core.post", - "pk": 3197, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.854Z", - "title": "Baby scooby in sad mood....", - "body": "
\"Baby
", - "author": "deepanshuahiroo7", - "publication_date": "2020-07-21T15:12:23Z", - "url": "https://www.reddit.com/r/aww/comments/hv73ft/baby_scooby_in_sad_mood/", - "read": false, - "rule": 81, - "remote_identifier": "hv73ft" - } -}, -{ - "model": "core.post", - "pk": 3198, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:14:50.856Z", - "title": "New friends!", - "body": "
\"New
", - "author": "HelentotheKeller", - "publication_date": "2020-07-21T13:10:48Z", - "url": "https://www.reddit.com/r/aww/comments/hv5i6i/new_friends/", - "read": false, - "rule": 81, - "remote_identifier": "hv5i6i" - } -}, -{ - "model": "core.post", - "pk": 3199, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:14:50.858Z", - "title": "When you haven't chewed anything for 1 second", - "body": "
\"When
", - "author": "Tanay4", - "publication_date": "2020-07-21T10:26:53Z", - "url": "https://www.reddit.com/r/aww/comments/hv3pl0/when_you_havent_chewed_anything_for_1_second/", - "read": false, - "rule": 81, - "remote_identifier": "hv3pl0" - } -}, -{ - "model": "core.post", - "pk": 3200, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:17:01.490Z", - "title": "Mango Derp", - "body": "
\"Mango
", - "author": "sheetglass", - "publication_date": "2020-07-21T13:27:26Z", - "url": "https://www.reddit.com/r/aww/comments/hv5p8s/mango_derp/", - "read": true, - "rule": 81, - "remote_identifier": "hv5p8s" - } -}, -{ - "model": "core.post", - "pk": 3201, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:14:50.863Z", - "title": "My guy turns 20 next month", - "body": "
\"My
", - "author": "alozsoc", - "publication_date": "2020-07-21T06:34:26Z", - "url": "https://www.reddit.com/r/aww/comments/hv0xp1/my_guy_turns_20_next_month/", - "read": false, - "rule": 81, - "remote_identifier": "hv0xp1" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "add_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "change_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "delete_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "view_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "add_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "change_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "delete_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "view_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add group", - "content_type": [ - "auth", - "group" - ], - "codename": "add_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change group", - "content_type": [ - "auth", - "group" - ], - "codename": "change_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete group", - "content_type": [ - "auth", - "group" - ], - "codename": "delete_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view group", - "content_type": [ - "auth", - "group" - ], - "codename": "view_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "add_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "change_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "delete_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "view_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add session", - "content_type": [ - "sessions", - "session" - ], - "codename": "add_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change session", - "content_type": [ - "sessions", - "session" - ], - "codename": "change_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete session", - "content_type": [ - "sessions", - "session" - ], - "codename": "delete_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view session", - "content_type": [ - "sessions", - "session" - ], - "codename": "view_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "add_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "change_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "delete_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "view_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "add_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "change_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "delete_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "view_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "add_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "change_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "delete_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "view_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "add_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "change_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "delete_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "view_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "add_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "change_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "delete_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "view_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "add_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "change_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "delete_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "view_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "add_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "change_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "delete_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "view_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "add_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "change_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "delete_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "view_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "add_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "change_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "delete_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "view_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "add_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "change_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "delete_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "view_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add user", - "content_type": [ - "accounts", - "user" - ], - "codename": "add_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change user", - "content_type": [ - "accounts", - "user" - ], - "codename": "change_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete user", - "content_type": [ - "accounts", - "user" - ], - "codename": "delete_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view user", - "content_type": [ - "accounts", - "user" - ], - "codename": "view_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add post", - "content_type": [ - "core", - "post" - ], - "codename": "add_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change post", - "content_type": [ - "core", - "post" - ], - "codename": "change_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete post", - "content_type": [ - "core", - "post" - ], - "codename": "delete_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view post", - "content_type": [ - "core", - "post" - ], - "codename": "view_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add Category", - "content_type": [ - "core", - "category" - ], - "codename": "add_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change Category", - "content_type": [ - "core", - "category" - ], - "codename": "change_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete Category", - "content_type": [ - "core", - "category" - ], - "codename": "delete_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view Category", - "content_type": [ - "core", - "category" - ], - "codename": "view_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "add_collectionrule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "change_collectionrule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "delete_collectionrule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "view_collectionrule" - } -}, -{ - "model": "accounts.user", - "fields": { - "password": "pbkdf2_sha256$180000$U9a2CS9X0b8Y$T6bD/VoUOFoGNIp16aFlOL0N7q0e6A3I97ypm/AhsGo=", - "last_login": "2020-07-21T20:14:35.966Z", - "is_superuser": true, - "first_name": "", - "last_name": "", - "is_staff": true, - "is_active": true, - "date_joined": "2019-07-18T18:52:36.080Z", - "email": "sonny@bakker.nl", - "task": 10, - "reddit_refresh_token": null, - "reddit_access_token": null, - "groups": [], - "user_permissions": [] - } -}, -{ - "model": "core.category", - "pk": 8, - "fields": { - "created": "2019-11-17T19:37:24.671Z", - "modified": "2019-11-18T19:59:55.010Z", - "name": "World news", - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "core.category", - "pk": 9, - "fields": { - "created": "2019-11-17T19:37:26.161Z", - "modified": "2020-05-30T13:36:10.509Z", - "name": "Tech", - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 3, - "fields": { - "created": "2019-07-14T13:08:10.374Z", - "modified": "2020-07-14T11:45:30.680Z", - "name": "Hackers News", - "type": "feed", - "url": "https://news.ycombinator.com/rss", - "website_url": "https://news.ycombinator.com/", - "favicon": "https://news.ycombinator.com/favicon.ico", - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:30.477Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 4, - "fields": { - "created": "2019-07-20T11:24:32.745Z", - "modified": "2020-07-14T11:45:29.357Z", - "name": "BBC", - "type": "feed", - "url": "http://feeds.bbci.co.uk/news/world/rss.xml", - "website_url": "https://www.bbc.co.uk/news/", - "favicon": "https://m.files.bbci.co.uk/modules/bbc-morph-news-waf-page-meta/2.5.2/apple-touch-icon-57x57-precomposed.png", - "timezone": "UTC", - "category": 8, - "last_suceeded": "2020-07-14T11:45:28.863Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 5, - "fields": { - "created": "2019-07-20T11:24:50.411Z", - "modified": "2020-07-14T11:45:30.063Z", - "name": "Ars Technica", - "type": "feed", - "url": "http://feeds.arstechnica.com/arstechnica/index?fmt=xml", - "website_url": "https://arstechnica.com", - "favicon": "https://cdn.arstechnica.net/favicon.ico", - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:29.810Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 6, - "fields": { - "created": "2019-07-20T11:25:02.089Z", - "modified": "2020-07-14T11:45:30.473Z", - "name": "The Guardian", - "type": "feed", - "url": "https://www.theguardian.com/world/rss", - "website_url": "https://www.theguardian.com/world", - "favicon": "https://assets.guim.co.uk/images/favicons/873381bf11d58e20f551905d51575117/72x72.png", - "timezone": "UTC", - "category": 8, - "last_suceeded": "2020-07-14T11:45:30.181Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 7, - "fields": { - "created": "2019-07-20T11:25:30.121Z", - "modified": "2020-07-14T11:45:29.807Z", - "name": "Tweakers", - "type": "feed", - "url": "http://feeds.feedburner.com/tweakers/mixed?fmt=xml", - "website_url": "https://tweakers.net/", - "favicon": null, - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:29.525Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 8, - "fields": { - "created": "2019-07-20T11:25:46.256Z", - "modified": "2020-07-14T11:45:30.179Z", - "name": "The Verge", - "type": "feed", - "url": "https://www.theverge.com/rss/index.xml", - "website_url": "https://www.theverge.com/", - "favicon": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395367/favicon-16x16.0.png", - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:30.066Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 9, - "fields": { - "created": "2019-11-24T15:28:41.399Z", - "modified": "2020-07-14T11:45:29.522Z", - "name": "NOS", - "type": "feed", - "url": "http://feeds.nos.nl/nosnieuwsalgemeen", - "website_url": null, - "favicon": null, - "timezone": "Europe/Amsterdam", - "category": 8, - "last_suceeded": "2020-07-14T11:45:29.362Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 80, - "fields": { - "created": "2020-07-08T19:30:10.638Z", - "modified": "2020-07-21T20:14:50.609Z", - "name": "Linux subreddit", - "type": "subreddit", - "url": "https://oauth.reddit.com/r/linux/hot", - "website_url": null, - "favicon": null, - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-21T20:14:50.492Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 81, - "fields": { - "created": "2020-07-08T19:30:33.590Z", - "modified": "2020-07-21T20:14:50.865Z", - "name": "AWW subreddit", - "type": "subreddit", - "url": "https://oauth.reddit.com/r/aww/hot", - "website_url": null, - "favicon": null, - "timezone": "UTC", - "category": 8, - "last_suceeded": "2020-07-21T20:14:50.768Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 82, - "fields": { - "created": "2020-07-20T19:29:37.675Z", - "modified": "2020-07-21T20:14:50.489Z", - "name": "Star citizen subreddit", - "type": "subreddit", - "url": "https://oauth.reddit.com/r/starcitizen/hot.json", - "website_url": null, - "favicon": null, - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-21T20:14:50.355Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "admin.logentry", - "pk": 1, - "fields": { - "action_time": "2020-05-24T18:38:44.624Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "object_id": "5", - "object_repr": "every 4 hours", - "action_flag": 1, - "change_message": "[{\"added\": {}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 2, - "fields": { - "action_time": "2020-05-24T18:38:46.689Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "10", - "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Interval Schedule\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 3, - "fields": { - "action_time": "2020-05-24T18:39:09.203Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "26", - "object_repr": "sonnyba871@gmail.com-collection-task: every hour", - "action_flag": 3, - "change_message": "" - } -}, -{ - "model": "admin.logentry", - "pk": 4, - "fields": { - "action_time": "2020-05-24T19:46:50.248Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "10", - "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Positional Arguments\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 5, - "fields": { - "action_time": "2020-07-07T19:37:57.086Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Reddit refresh token\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 6, - "fields": { - "action_time": "2020-07-07T19:39:46.160Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "10", - "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Task (registered)\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 7, - "fields": { - "action_time": "2020-07-08T19:29:27.025Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "11", - "object_repr": "Reddit collection task: every 4 hours", - "action_flag": 1, - "change_message": "[{\"added\": {}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 8, - "fields": { - "action_time": "2020-07-14T11:46:50.039Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 9, - "fields": { - "action_time": "2020-07-18T19:08:33.997Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "collection", - "collectionrule" - ], - "object_id": "81", - "object_repr": "AWW subreddit", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 10, - "fields": { - "action_time": "2020-07-18T19:08:44.063Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "collection", - "collectionrule" - ], - "object_id": "80", - "object_repr": "Linux subreddit", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 11, - "fields": { - "action_time": "2020-07-18T19:17:25.213Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2336", - "object_repr": "Post-2336", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 12, - "fields": { - "action_time": "2020-07-18T19:17:40.596Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2336", - "object_repr": "Post-2336", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 13, - "fields": { - "action_time": "2020-07-19T10:55:55.807Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2764", - "object_repr": "Post-2764", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 14, - "fields": { - "action_time": "2020-07-19T10:57:40.643Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2764", - "object_repr": "Post-2764", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 15, - "fields": { - "action_time": "2020-07-19T10:58:05.823Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2764", - "object_repr": "Post-2764", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 16, - "fields": { - "action_time": "2020-07-26T09:51:52.478Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 17, - "fields": { - "action_time": "2020-07-26T09:52:04.691Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"password\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 18, - "fields": { - "action_time": "2020-07-26T09:52:12.392Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 19, - "fields": { - "action_time": "2020-07-26T09:56:15.949Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" - } -} -] +[ +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "admin", + "model": "logentry" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", + "model": "permission" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", + "model": "group" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "contenttypes", + "model": "contenttype" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "sessions", + "model": "session" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "crontabschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "intervalschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "periodictask" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "periodictasks" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "solarschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "clockedschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "registration", + "model": "registrationprofile" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "registration", + "model": "supervisedregistrationprofile" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "axes", + "model": "accessattempt" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "axes", + "model": "accesslog" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "accounts", + "model": "user" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "core", + "model": "post" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "core", + "model": "category" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "collection", + "model": "collectionrule" + } +}, +{ + "model": "sessions.session", + "pk": "3sumq22krk8tsvexcs4b8czu82yhvuer", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-05-16T18:29:04.049Z" + } +}, +{ + "model": "sessions.session", + "pk": "8ix6bdwf2ywk0eir1hb062dhfh9xit85", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-07-21T19:36:54.530Z" + } +}, +{ + "model": "sessions.session", + "pk": "d4wophwpjm8z96doe8iddvhdv9yfafyx", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-06-07T19:45:49.727Z" + } +}, +{ + "model": "sessions.session", + "pk": "g23ziz66li5zx8nd8cewb3vevdxhjkm0", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-06-30T06:55:50.747Z" + } +}, +{ + "model": "sessions.session", + "pk": "jwn66dptmdkm6hom2ns3j288aaxqtyjd", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-06-07T18:38:19.116Z" + } +}, +{ + "model": "sessions.session", + "pk": "wjz6kwg5e5ciemre0l0wwyrcwcj2gyg6", + "fields": { + "session_data": "MWU5ODBjY2QyOTFhMmRiY2QyYjQwZjQ3MmMwYmExYjBlYTkxNTcwODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiI0YWZkYTkxNzU5ZDBhZDZmMjg1ZTQyOGY0OTUxN2M5MTJhMmM5NWIyIn0=", + "expire_date": "2020-08-09T09:52:04.705Z" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 1, + "fields": { + "every": 5, + "period": "minutes" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 2, + "fields": { + "every": 15, + "period": "minutes" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 3, + "fields": { + "every": 30, + "period": "minutes" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 4, + "fields": { + "every": 1, + "period": "hours" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 5, + "fields": { + "every": 4, + "period": "hours" + } +}, +{ + "model": "django_celery_beat.crontabschedule", + "pk": 1, + "fields": { + "minute": "0", + "hour": "4", + "day_of_week": "*", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "UTC" + } +}, +{ + "model": "django_celery_beat.periodictasks", + "pk": 1, + "fields": { + "last_update": "2020-07-26T09:47:48.298Z" + } +}, +{ + "model": "django_celery_beat.periodictask", + "pk": 1, + "fields": { + "name": "celery.backend_cleanup", + "task": "celery.backend_cleanup", + "interval": null, + "crontab": 1, + "solar": null, + "clocked": null, + "args": "[]", + "kwargs": "{}", + "queue": null, + "exchange": null, + "routing_key": null, + "headers": "{}", + "priority": null, + "expires": null, + "expire_seconds": 43200, + "one_off": false, + "start_time": null, + "enabled": true, + "last_run_at": "2020-07-26T09:47:48.322Z", + "total_run_count": 17, + "date_changed": "2020-07-26T09:47:50.362Z", + "description": "" + } +}, +{ + "model": "django_celery_beat.periodictask", + "pk": 10, + "fields": { + "name": "sonny@bakker.nl-collection-task", + "task": "FeedTask", + "interval": 5, + "crontab": null, + "solar": null, + "clocked": null, + "args": "[1]", + "kwargs": "{}", + "queue": null, + "exchange": null, + "routing_key": null, + "headers": "{}", + "priority": null, + "expires": null, + "expire_seconds": null, + "one_off": false, + "start_time": null, + "enabled": false, + "last_run_at": "2020-07-14T11:45:26.209Z", + "total_run_count": 307, + "date_changed": "2020-07-14T11:45:41.282Z", + "description": "" + } +}, +{ + "model": "django_celery_beat.periodictask", + "pk": 11, + "fields": { + "name": "Reddit collection task", + "task": "RedditTask", + "interval": 5, + "crontab": null, + "solar": null, + "clocked": null, + "args": "[]", + "kwargs": "{}", + "queue": null, + "exchange": null, + "routing_key": null, + "headers": "{}", + "priority": null, + "expires": null, + "expire_seconds": null, + "one_off": false, + "start_time": null, + "enabled": false, + "last_run_at": null, + "total_run_count": 4, + "date_changed": "2020-07-14T11:45:41.316Z", + "description": "" + } +}, +{ + "model": "core.post", + "pk": 3061, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:14:50.423Z", + "title": "Star Citizen: Question and Answer Thread", + "body": "

Welcome to the Star Citizen question and answer thread. Feel free to ask any questions you have related to SC here!

\n\n\n\n

Useful Links and Resources:

\n\n

Star Citizen Wiki - The biggest and best wiki resource dedicated to Star Citizen

\n\n

Star Citizen FAQ - Chances the answer you need is here.

\n\n

Discord Help Channel - Often times community members will be here to help you with issues.

\n\n

Referral Code Randomizer - Use this when creating a new account to get 5000 extra UEC.

\n\n

Download Star Citizen - Get the latest version of Star Citizen here

\n\n

Current Game Features - Click here to see what you can currently do in Star Citizen.

\n\n

Development Roadmap - The current development status of up and coming Star Citizen features.

\n\n

Pledge FAQ - Official FAQ regarding spending money on the game.

\n
", + "author": "UEE_Central_Computer", + "publication_date": "2020-07-20T14:00:10Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huk04t/star_citizen_question_and_answer_thread/", + "read": false, + "rule": 82, + "remote_identifier": "huk04t" + } +}, +{ + "model": "core.post", + "pk": 3062, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:33:37.019Z", + "title": "Peace and Quiet", + "body": "
\"Peace
", + "author": "SourMemeNZ", + "publication_date": "2020-07-20T14:09:49Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huk4ib/peace_and_quiet/", + "read": true, + "rule": 82, + "remote_identifier": "huk4ib" + } +}, +{ + "model": "core.post", + "pk": 3063, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:14:50.463Z", + "title": "Y'all are probably sick of em by now but here's my LEGO Mercury Star Runner (MSR).", + "body": "
\"Y'all
", + "author": "osamadabinman", + "publication_date": "2020-07-20T19:53:23Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hupzqa/yall_are_probably_sick_of_em_by_now_but_heres_my/", + "read": true, + "rule": 82, + "remote_identifier": "hupzqa" + } +}, +{ + "model": "core.post", + "pk": 3064, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:12.253Z", + "title": "Damned Space Invaders and their pixel weapons!", + "body": "
\"Damned
", + "author": "Akaradrin", + "publication_date": "2020-07-20T14:26:18Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hukckf/damned_space_invaders_and_their_pixel_weapons/", + "read": true, + "rule": 82, + "remote_identifier": "hukckf" + } +}, +{ + "model": "core.post", + "pk": 3065, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.578Z", + "title": "The sky is no longer the limit", + "body": "
\"The
", + "author": "CyberTill", + "publication_date": "2020-07-20T14:11:31Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huk5b8/the_sky_is_no_longer_the_limit/", + "read": false, + "rule": 82, + "remote_identifier": "huk5b8" + } +}, +{ + "model": "core.post", + "pk": 3066, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:23.282Z", + "title": "Terrapin Hover Mode Gameplay [Full Video in Comments]", + "body": "
", + "author": "Didactic_Tomato", + "publication_date": "2020-07-20T11:01:13Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hui1gv/terrapin_hover_mode_gameplay_full_video_in/", + "read": true, + "rule": 82, + "remote_identifier": "hui1gv" + } +}, +{ + "model": "core.post", + "pk": 3067, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:44.250Z", + "title": "honestly", + "body": "
\"honestly\"
", + "author": "Beatlead", + "publication_date": "2020-07-20T18:24:07Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huo96t/honestly/", + "read": true, + "rule": 82, + "remote_identifier": "huo96t" + } +}, +{ + "model": "core.post", + "pk": 3068, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.584Z", + "title": "As a paranoiac and tired of checking if door was closed, saved to f4 thoses \"security cam\" positions, could be usefull for larger ships :)", + "body": "
Direct url
", + "author": "icwiener__", + "publication_date": "2020-07-20T13:03:33Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hujchz/as_a_paranoiac_and_tired_of_checking_if_door_was/", + "read": false, + "rule": 82, + "remote_identifier": "hujchz" + } +}, +{ + "model": "core.post", + "pk": 3069, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:33:59.158Z", + "title": "Station Manager: \"You're too fat, we won't let you in, go and fall on Lorville. Thank you for your call!\" Me: \"okay :'(\"", + "body": "
\"Station
", + "author": "Shaman_N_One", + "publication_date": "2020-07-20T11:33:38Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huidlu/station_manager_youre_too_fat_we_wont_let_you_in/", + "read": true, + "rule": 82, + "remote_identifier": "huidlu" + } +}, +{ + "model": "core.post", + "pk": 3070, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.588Z", + "title": "[PTU Bug Hunt Request] Packet Loss", + "body": "
Direct url
", + "author": "Rainwalker007", + "publication_date": "2020-07-20T18:38:03Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huoicq/ptu_bug_hunt_request_packet_loss/", + "read": false, + "rule": 82, + "remote_identifier": "huoicq" + } +}, +{ + "model": "core.post", + "pk": 3071, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:52.092Z", + "title": "Anyone able to explain these \"trail frames\"?", + "body": "
\"Anyone
", + "author": "Abnormal_Sloth", + "publication_date": "2020-07-20T17:11:32Z", + "url": "https://www.reddit.com/r/starcitizen/comments/humyeq/anyone_able_to_explain_these_trail_frames/", + "read": true, + "rule": 82, + "remote_identifier": "humyeq" + } +}, +{ + "model": "core.post", + "pk": 3072, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.593Z", + "title": "#BringBackBugSmasher - A long forgotten legendary video content", + "body": "
Direct url
", + "author": "MasterBoring", + "publication_date": "2020-07-20T18:05:54Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hunx77/bringbackbugsmasher_a_long_forgotten_legendary/", + "read": false, + "rule": 82, + "remote_identifier": "hunx77" + } +}, +{ + "model": "core.post", + "pk": 3073, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:33:22.601Z", + "title": "Oracle Helmet [in-game screenshot; downsampled to 4k]", + "body": "
\"Oracle
", + "author": "mr-hasgaha", + "publication_date": "2020-07-20T17:39:34Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hung0b/oracle_helmet_ingame_screenshot_downsampled_to_4k/", + "read": true, + "rule": 82, + "remote_identifier": "hung0b" + } +}, +{ + "model": "core.post", + "pk": 3074, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:34:42.578Z", + "title": "Testing 3.10 - Gladius in decoupled mode", + "body": "
", + "author": "DarkConstant", + "publication_date": "2020-07-19T21:26:52Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hu6f1h/testing_310_gladius_in_decoupled_mode/", + "read": true, + "rule": 82, + "remote_identifier": "hu6f1h" + } +}, +{ + "model": "core.post", + "pk": 3075, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:34:29.424Z", + "title": "Day 3, I can't stop taking pictures with my Carrack. Send help", + "body": "
\"Day
", + "author": "CyberTill", + "publication_date": "2020-07-20T01:58:15Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huazyy/day_3_i_cant_stop_taking_pictures_with_my_carrack/", + "read": true, + "rule": 82, + "remote_identifier": "huazyy" + } +}, +{ + "model": "core.post", + "pk": 3076, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.602Z", + "title": "I used to enjoy flying between the buildings of new babbage, I mean before the NFZ \"improvement\"", + "body": "
\"I
", + "author": "shoeii", + "publication_date": "2020-07-20T16:40:26Z", + "url": "https://www.reddit.com/r/starcitizen/comments/humet2/i_used_to_enjoy_flying_between_the_buildings_of/", + "read": false, + "rule": 82, + "remote_identifier": "humet2" + } +}, +{ + "model": "core.post", + "pk": 3077, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:18:04.237Z", + "title": "Thank you CIG for updated heightmaps and render distances", + "body": "
\"Thank
", + "author": "u7f76", + "publication_date": "2020-07-19T23:38:22Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hu8pwf/thank_you_cig_for_updated_heightmaps_and_render/", + "read": true, + "rule": 82, + "remote_identifier": "hu8pwf" + } +}, +{ + "model": "core.post", + "pk": 3078, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.607Z", + "title": "This Week in Star Citizen | July 20th 2020", + "body": "
Direct url
", + "author": "ivtiprogamer", + "publication_date": "2020-07-20T19:50:29Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hupxnt/this_week_in_star_citizen_july_20th_2020/", + "read": false, + "rule": 82, + "remote_identifier": "hupxnt" + } +}, +{ + "model": "core.post", + "pk": 3079, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:34:36.068Z", + "title": "Bravo CIG lighting team! Noticeable improvements to all around environment lighting in 3.10", + "body": "
\"Bravo
", + "author": "u7f76", + "publication_date": "2020-07-20T00:02:23Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hu94o0/bravo_cig_lighting_team_noticeable_improvements/", + "read": true, + "rule": 82, + "remote_identifier": "hu94o0" + } +}, +{ + "model": "core.post", + "pk": 3080, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.613Z", + "title": "Thick", + "body": "
\"Thick\"
", + "author": "burgerbagel", + "publication_date": "2020-07-20T16:24:38Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hum50f/thick/", + "read": false, + "rule": 82, + "remote_identifier": "hum50f" + } +}, +{ + "model": "core.post", + "pk": 3081, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:34:19.763Z", + "title": "Soon\u2122", + "body": "
\"Soon\u2122\"
", + "author": "Mistralette", + "publication_date": "2020-07-20T05:54:09Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hueg01/soon/", + "read": true, + "rule": 82, + "remote_identifier": "hueg01" + } +}, +{ + "model": "core.post", + "pk": 3082, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.618Z", + "title": "On the prowl", + "body": "
\"On
", + "author": "SaraCaterina", + "publication_date": "2020-07-20T16:37:03Z", + "url": "https://www.reddit.com/r/starcitizen/comments/humcmb/on_the_prowl/", + "read": false, + "rule": 82, + "remote_identifier": "humcmb" + } +}, +{ + "model": "core.post", + "pk": 3083, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:34:07.272Z", + "title": "The Hills Have Eyes", + "body": "
\"The
", + "author": "FallenLordik", + "publication_date": "2020-07-20T11:19:19Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hui8ao/the_hills_have_eyes/", + "read": true, + "rule": 82, + "remote_identifier": "hui8ao" + } +}, +{ + "model": "core.post", + "pk": 3084, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.623Z", + "title": "Worried about longer loading screens? Hit ~ and do r_displayinfo 3", + "body": "
\"Worried
", + "author": "kristokn", + "publication_date": "2020-07-20T10:09:53Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huhif1/worried_about_longer_loading_screens_hit_and_do_r/", + "read": false, + "rule": 82, + "remote_identifier": "huhif1" + } +}, +{ + "model": "core.post", + "pk": 3085, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.625Z", + "title": "My contribution to the wallpaper contest... click for the full effect (3440x1440)", + "body": "
\"My
", + "author": "Dougie_Juice", + "publication_date": "2020-07-20T20:02:31Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huq655/my_contribution_to_the_wallpaper_contest_click/", + "read": false, + "rule": 82, + "remote_identifier": "huq655" + } +}, +{ + "model": "core.post", + "pk": 3086, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.627Z", + "title": "Star Citizen: The Onion (Parody Project)", + "body": "
Direct url
", + "author": "BroadOne", + "publication_date": "2020-07-20T19:19:20Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hupbkj/star_citizen_the_onion_parody_project/", + "read": false, + "rule": 82, + "remote_identifier": "hupbkj" + } +}, +{ + "model": "core.post", + "pk": 3087, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.637Z", + "title": "perfect day to sunbathe", + "body": "
", + "author": "Pedrica1", + "publication_date": "2020-07-20T18:08:17Z", + "url": "https://www.reddit.com/r/aww/comments/hunysb/perfect_day_to_sunbathe/", + "read": false, + "rule": 81, + "remote_identifier": "hunysb" + } +}, +{ + "model": "core.post", + "pk": 3088, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.639Z", + "title": "My dogs face when he sees I'm home", + "body": "
", + "author": "NewReddit_WhoDis", + "publication_date": "2020-07-20T16:45:21Z", + "url": "https://www.reddit.com/r/aww/comments/humhxa/my_dogs_face_when_he_sees_im_home/", + "read": false, + "rule": 81, + "remote_identifier": "humhxa" + } +}, +{ + "model": "core.post", + "pk": 3089, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.641Z", + "title": "Cow loves the scritch machine", + "body": "
", + "author": "Der_Ist", + "publication_date": "2020-07-20T17:36:16Z", + "url": "https://www.reddit.com/r/aww/comments/hundvo/cow_loves_the_scritch_machine/", + "read": false, + "rule": 81, + "remote_identifier": "hundvo" + } +}, +{ + "model": "core.post", + "pk": 3090, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.643Z", + "title": "Can I sit next to you ?", + "body": "
", + "author": "wheezy098", + "publication_date": "2020-07-20T17:55:10Z", + "url": "https://www.reddit.com/r/aww/comments/hunq5h/can_i_sit_next_to_you/", + "read": false, + "rule": 81, + "remote_identifier": "hunq5h" + } +}, +{ + "model": "core.post", + "pk": 3091, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.645Z", + "title": "IS THAT A CUSTOMER? flop flop flop flop .... \" Can I uhh... help you sir?\"", + "body": "
", + "author": "MBMV", + "publication_date": "2020-07-20T12:50:40Z", + "url": "https://www.reddit.com/r/aww/comments/huj7g3/is_that_a_customer_flop_flop_flop_flop_can_i_uhh/", + "read": false, + "rule": 81, + "remote_identifier": "huj7g3" + } +}, +{ + "model": "core.post", + "pk": 3092, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.647Z", + "title": "Good Boy turned Disney Princess", + "body": "
", + "author": "Sauwercraud", + "publication_date": "2020-07-20T18:40:05Z", + "url": "https://www.reddit.com/r/aww/comments/huojq0/good_boy_turned_disney_princess/", + "read": false, + "rule": 81, + "remote_identifier": "huojq0" + } +}, +{ + "model": "core.post", + "pk": 3093, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.649Z", + "title": "Kitty loop", + "body": "
", + "author": "Dlatrex", + "publication_date": "2020-07-20T12:54:02Z", + "url": "https://www.reddit.com/r/aww/comments/huj8s6/kitty_loop/", + "read": false, + "rule": 81, + "remote_identifier": "huj8s6" + } +}, +{ + "model": "core.post", + "pk": 3094, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.652Z", + "title": "if i fits i sits", + "body": "
", + "author": "jasontaken", + "publication_date": "2020-07-20T16:38:32Z", + "url": "https://www.reddit.com/r/aww/comments/humdlf/if_i_fits_i_sits/", + "read": false, + "rule": 81, + "remote_identifier": "humdlf" + } +}, +{ + "model": "core.post", + "pk": 3095, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.654Z", + "title": "Isn\u2019t she Adorable !", + "body": "
\"Isn\u2019t
", + "author": "MunchyMac", + "publication_date": "2020-07-20T16:18:05Z", + "url": "https://www.reddit.com/r/aww/comments/hum133/isnt_she_adorable/", + "read": false, + "rule": 81, + "remote_identifier": "hum133" + } +}, +{ + "model": "core.post", + "pk": 3096, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.655Z", + "title": "Thank you mama (\u2283\uff61\u2022\u0301\u203f\u2022\u0300\uff61)\u2283", + "body": "
", + "author": "AnoushkaSingh", + "publication_date": "2020-07-20T13:35:51Z", + "url": "https://www.reddit.com/r/aww/comments/hujpxy/thank_you_mama/", + "read": false, + "rule": 81, + "remote_identifier": "hujpxy" + } +}, +{ + "model": "core.post", + "pk": 3097, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.657Z", + "title": "I WANT TO HUG HIM SO BAD!!!", + "body": "
", + "author": "BATMAN_5777", + "publication_date": "2020-07-20T18:25:20Z", + "url": "https://www.reddit.com/r/aww/comments/huo9z4/i_want_to_hug_him_so_bad/", + "read": false, + "rule": 81, + "remote_identifier": "huo9z4" + } +}, +{ + "model": "core.post", + "pk": 3098, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.659Z", + "title": "Before and after being called a good boy", + "body": "
\"Before
", + "author": "vladgrinch", + "publication_date": "2020-07-20T10:48:40Z", + "url": "https://www.reddit.com/r/aww/comments/huhwu9/before_and_after_being_called_a_good_boy/", + "read": false, + "rule": 81, + "remote_identifier": "huhwu9" + } +}, +{ + "model": "core.post", + "pk": 3099, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.662Z", + "title": "My fianc\u00e9 has wanted a dog his whole life. This is his college graduation present. Welcome home Maple!", + "body": "
\"My
", + "author": "AlexisaurusRex", + "publication_date": "2020-07-20T17:57:25Z", + "url": "https://www.reddit.com/r/aww/comments/hunrie/my_fianc\u00e9_has_wanted_a_dog_his_whole_life_this_is/", + "read": false, + "rule": 81, + "remote_identifier": "hunrie" + } +}, +{ + "model": "core.post", + "pk": 3100, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.664Z", + "title": "Cute burro.", + "body": "
\"Cute
", + "author": "Craftmine101", + "publication_date": "2020-07-20T13:45:32Z", + "url": "https://www.reddit.com/r/aww/comments/huju40/cute_burro/", + "read": false, + "rule": 81, + "remote_identifier": "huju40" + } +}, +{ + "model": "core.post", + "pk": 3101, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.666Z", + "title": "I've never seen anyone dance better than that turtle.", + "body": "
", + "author": "Ashley1023", + "publication_date": "2020-07-20T18:07:30Z", + "url": "https://www.reddit.com/r/aww/comments/hunya8/ive_never_seen_anyone_dance_better_than_that/", + "read": false, + "rule": 81, + "remote_identifier": "hunya8" + } +}, +{ + "model": "core.post", + "pk": 3102, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.669Z", + "title": "Someone\u2019s going to be quite surprised when he realizes all this new stuff isn\u2019t for him!", + "body": "
\"Someone\u2019s
", + "author": "molly590", + "publication_date": "2020-07-20T15:46:21Z", + "url": "https://www.reddit.com/r/aww/comments/hulikg/someones_going_to_be_quite_surprised_when_he/", + "read": false, + "rule": 81, + "remote_identifier": "hulikg" + } +}, +{ + "model": "core.post", + "pk": 3103, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.671Z", + "title": "my aunt asked me to paint her puppy and I think it turned out so cute!!!", + "body": "
\"my
", + "author": "PineappleLightt", + "publication_date": "2020-07-20T16:39:37Z", + "url": "https://www.reddit.com/r/aww/comments/humea0/my_aunt_asked_me_to_paint_her_puppy_and_i_think/", + "read": false, + "rule": 81, + "remote_identifier": "humea0" + } +}, +{ + "model": "core.post", + "pk": 3104, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.673Z", + "title": "Master Assassin", + "body": "
\"Master
", + "author": "LauWalker", + "publication_date": "2020-07-20T18:47:52Z", + "url": "https://www.reddit.com/r/aww/comments/huop8a/master_assassin/", + "read": false, + "rule": 81, + "remote_identifier": "huop8a" + } +}, +{ + "model": "core.post", + "pk": 3105, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.675Z", + "title": "Every time this tank cleaner cleans out the aquarium, this fish swims over to him looking for pets", + "body": "
Direct url
", + "author": "unnaturalorder", + "publication_date": "2020-07-20T05:29:30Z", + "url": "https://www.reddit.com/r/aww/comments/hue3r0/every_time_this_tank_cleaner_cleans_out_the/", + "read": false, + "rule": 81, + "remote_identifier": "hue3r0" + } +}, +{ + "model": "core.post", + "pk": 3106, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.678Z", + "title": "My girlfriend sent me this while I was at work. And here I was thinking the perfect picture of our dog didn't exist", + "body": "
Direct url
", + "author": "Khuma-zi_Eldrama", + "publication_date": "2020-07-20T19:22:48Z", + "url": "https://www.reddit.com/r/aww/comments/hupdz8/my_girlfriend_sent_me_this_while_i_was_at_work/", + "read": false, + "rule": 81, + "remote_identifier": "hupdz8" + } +}, +{ + "model": "core.post", + "pk": 3107, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.680Z", + "title": "My first ever post, everyone meet my new baby girl Kiora! I\u2019m so in love with her\ud83e\udd7a\ud83d\udcab", + "body": "
\"My
", + "author": "Dumpling2463", + "publication_date": "2020-07-20T05:34:29Z", + "url": "https://www.reddit.com/r/aww/comments/hue6dx/my_first_ever_post_everyone_meet_my_new_baby_girl/", + "read": false, + "rule": 81, + "remote_identifier": "hue6dx" + } +}, +{ + "model": "core.post", + "pk": 3108, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.682Z", + "title": "Dog splashing in water", + "body": "
Direct url
", + "author": "TheRikari", + "publication_date": "2020-07-20T15:44:02Z", + "url": "https://www.reddit.com/r/aww/comments/hulh8k/dog_splashing_in_water/", + "read": false, + "rule": 81, + "remote_identifier": "hulh8k" + } +}, +{ + "model": "core.post", + "pk": 3109, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.685Z", + "title": "They say taking breaks is the key to productivity!", + "body": "
", + "author": "Thereaper29", + "publication_date": "2020-07-20T05:43:40Z", + "url": "https://www.reddit.com/r/aww/comments/hueawt/they_say_taking_breaks_is_the_key_to_productivity/", + "read": false, + "rule": 81, + "remote_identifier": "hueawt" + } +}, +{ + "model": "core.post", + "pk": 3110, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.687Z", + "title": "I went away for 3 weeks, and now my cat is in love with my husband", + "body": "
\"I
", + "author": "sillykittyish", + "publication_date": "2020-07-20T03:29:11Z", + "url": "https://www.reddit.com/r/aww/comments/hucd7u/i_went_away_for_3_weeks_and_now_my_cat_is_in_love/", + "read": false, + "rule": 81, + "remote_identifier": "hucd7u" + } +}, +{ + "model": "core.post", + "pk": 3111, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.689Z", + "title": "Can you feel the love", + "body": "
", + "author": "kettySewrdPic", + "publication_date": "2020-07-20T09:13:32Z", + "url": "https://www.reddit.com/r/aww/comments/hugx1k/can_you_feel_the_love/", + "read": false, + "rule": 81, + "remote_identifier": "hugx1k" + } +}, +{ + "model": "core.post", + "pk": 3112, + "fields": { + "created": "2020-07-20T19:32:35.835Z", + "modified": "2020-07-21T20:14:50.522Z", + "title": "Linux Experiences/Rants or Education/Certifications thread - July 20, 2020", + "body": "

Welcome to r/linux rants and experiences! This megathread is also to hear opinions from anyone just starting out with Linux or those that have used Linux (GNU or otherwise) for a long time.

\n\n

Let us know what's annoying you, whats making you happy, or something that you want to get out to r/linux but didn't make the cut into a full post of it's own.

\n\n

For those looking for certifications please use this megathread to ask about how to get certified whether it's for the business world or for your own satisfaction. Be sure to check out r/linuxadmin for more discussion in the SysAdmin world!

\n\n

Please keep questions in r/linuxquestions, r/linux4noobs, or the Wednesday automod thread.

\n
", + "author": "AutoModerator", + "publication_date": "2020-07-20T06:12:00Z", + "url": "https://www.reddit.com/r/linux/comments/hueoo0/linux_experiencesrants_or_educationcertifications/", + "read": false, + "rule": 80, + "remote_identifier": "hueoo0" + } +}, +{ + "model": "core.post", + "pk": 3113, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:19:49.339Z", + "title": "Unix Family Tree", + "body": "
\"Unix
", + "author": "bauripalash", + "publication_date": "2020-07-20T10:32:15Z", + "url": "https://www.reddit.com/r/linux/comments/huhqrh/unix_family_tree/", + "read": true, + "rule": 80, + "remote_identifier": "huhqrh" + } +}, +{ + "model": "core.post", + "pk": 3114, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:14:50.554Z", + "title": "NVIDIA open sourced part of NVAPI SDK to aid 'Windows emulation environments'", + "body": "
Direct url
", + "author": "ignapk", + "publication_date": "2020-07-20T13:17:19Z", + "url": "https://www.reddit.com/r/linux/comments/huji8c/nvidia_open_sourced_part_of_nvapi_sdk_to_aid/", + "read": false, + "rule": 80, + "remote_identifier": "huji8c" + } +}, +{ + "model": "core.post", + "pk": 3115, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:14:50.551Z", + "title": "Jellyfin 10.6 released", + "body": "
Direct url
", + "author": "resoluti0n_", + "publication_date": "2020-07-20T16:40:05Z", + "url": "https://www.reddit.com/r/linux/comments/humekr/jellyfin_106_released/", + "read": false, + "rule": 80, + "remote_identifier": "humekr" + } +}, +{ + "model": "core.post", + "pk": 3116, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:14:50.583Z", + "title": "[German] Article in major german newspaper about trying Linux and WSL. Literal: \"Why it's beneficial to try Linux now\"", + "body": "
Direct url
", + "author": "noname7890", + "publication_date": "2020-07-19T15:19:27Z", + "url": "https://www.reddit.com/r/linux/comments/hu0d5v/german_article_in_major_german_newspaper_about/", + "read": false, + "rule": 80, + "remote_identifier": "hu0d5v" + } +}, +{ + "model": "core.post", + "pk": 3117, + "fields": { + "created": "2020-07-20T19:32:35.837Z", + "modified": "2020-07-21T20:14:50.574Z", + "title": "Brian Kernighan: UNIX, C, AWK, AMPL, and Go Programming | AI Podcast #109 with Lex Fridman", + "body": "
Direct url
", + "author": "tinyatom", + "publication_date": "2020-07-20T08:48:35Z", + "url": "https://www.reddit.com/r/linux/comments/hugn0w/brian_kernighan_unix_c_awk_ampl_and_go/", + "read": false, + "rule": 80, + "remote_identifier": "hugn0w" + } +}, +{ + "model": "core.post", + "pk": 3118, + "fields": { + "created": "2020-07-20T19:32:35.837Z", + "modified": "2020-07-21T20:14:50.578Z", + "title": "Explaining Computers Host Christopher Barnatt Has Switched To Linux", + "body": "
Direct url
", + "author": "sysrpl", + "publication_date": "2020-07-20T13:00:02Z", + "url": "https://www.reddit.com/r/linux/comments/hujb12/explaining_computers_host_christopher_barnatt_has/", + "read": false, + "rule": 80, + "remote_identifier": "hujb12" + } +}, +{ + "model": "core.post", + "pk": 3119, + "fields": { + "created": "2020-07-20T19:32:35.837Z", + "modified": "2020-07-21T20:14:50.529Z", + "title": "Ireland donates contact tracing app to the Linux foundation.", + "body": "
Direct url
", + "author": "mathiasryan", + "publication_date": "2020-07-20T21:31:43Z", + "url": "https://www.reddit.com/r/linux/comments/hury4e/ireland_donates_contact_tracing_app_to_the_linux/", + "read": false, + "rule": 80, + "remote_identifier": "hury4e" + } +}, +{ + "model": "core.post", + "pk": 3120, + "fields": { + "created": "2020-07-20T19:32:35.842Z", + "modified": "2020-07-21T20:14:50.588Z", + "title": "I implemented a simple terminal-based password manager", + "body": "

I created a simple, secure, and free password manager written in C: SaltPass. I haven't contributed open source code before, but I think this might be useful to a few people. Especially as an alternative to paid solutions such as LastPass and the likes. Any suggestions/edits/code improvements would be greatly appreciated!

\n
", + "author": "zaid-gg", + "publication_date": "2020-07-20T07:43:03Z", + "url": "https://www.reddit.com/r/linux/comments/hufula/i_implemented_a_simple_terminalbased_password/", + "read": false, + "rule": 80, + "remote_identifier": "hufula" + } +}, +{ + "model": "core.post", + "pk": 3121, + "fields": { + "created": "2020-07-20T19:32:35.843Z", + "modified": "2020-07-21T20:14:50.593Z", + "title": "Performance analysis of multi services on container Docker, LXC, and LXD - Bulletin of Electrical Engineering and Informatics, Adinda Riztia Putri, Rendy Munadi, Ridha Muldina Negara Adaptive Network\u2026", + "body": "
Direct url
", + "author": "bmullan", + "publication_date": "2020-07-20T11:35:59Z", + "url": "https://www.reddit.com/r/linux/comments/huieio/performance_analysis_of_multi_services_on/", + "read": false, + "rule": 80, + "remote_identifier": "huieio" + } +}, +{ + "model": "core.post", + "pk": 3122, + "fields": { + "created": "2020-07-20T19:32:35.844Z", + "modified": "2020-07-21T20:14:50.602Z", + "title": "Create an Internal PKI using OpenSSL and NitroKey HSM", + "body": "
Direct url
", + "author": "PixelPaulaus", + "publication_date": "2020-07-20T06:18:41Z", + "url": "https://www.reddit.com/r/linux/comments/huerpn/create_an_internal_pki_using_openssl_and_nitrokey/", + "read": false, + "rule": 80, + "remote_identifier": "huerpn" + } +}, +{ + "model": "core.post", + "pk": 3123, + "fields": { + "created": "2020-07-20T19:32:35.844Z", + "modified": "2020-07-20T19:32:35.883Z", + "title": "vopono - run applications via VPNs with temporary network namespaces", + "body": "
Direct url
", + "author": "nivenkos", + "publication_date": "2020-07-19T20:02:57Z", + "url": "https://www.reddit.com/r/linux/comments/hu4vge/vopono_run_applications_via_vpns_with_temporary/", + "read": false, + "rule": 80, + "remote_identifier": "hu4vge" + } +}, +{ + "model": "core.post", + "pk": 3124, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.886Z", + "title": "Double (triple, quadruple...) internet speed with openvpn tap channel bonding to a linux VPS", + "body": "

I have been working a couple of days on my latest video about channel bonding - the video is heavily inspired be this article on Serverfault. In essence, I have been searching for a while on how to bond multiple VPN channels together in order to increase internet speed - there does not seem to be a lot of information around - mainly articles on forums and reddit state that it should be possible but a detailed guide is hard to find. I am using two Ubuntu machines in order to build the connection - one local and one VPS. The bash scripts I use in my video in order to achieve tap channel bonding are available on my github repository. I am currently working on a second video in order to walk through and explain the scripts in depth. Enjoy!

\n\n

(EDIT) - the question has come up in the discussions below if this is really packet load balancing or rather balancing links only - please see my comment further down - I can confirm that this DOES packet balancing so it does work as described.

\n
", + "author": "onemarcfifty", + "publication_date": "2020-07-19T20:41:40Z", + "url": "https://www.reddit.com/r/linux/comments/hu5l4f/double_triple_quadruple_internet_speed_with/", + "read": false, + "rule": 80, + "remote_identifier": "hu5l4f" + } +}, +{ + "model": "core.post", + "pk": 3125, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.888Z", + "title": "OpenRGB - Open source RGB lighting control that doesn't depend on manufacturer software, supports Linux", + "body": "
Direct url
", + "author": "pr0_c0d3", + "publication_date": "2020-07-18T16:52:48Z", + "url": "https://www.reddit.com/r/linux/comments/hthuli/openrgb_open_source_rgb_lighting_control_that/", + "read": false, + "rule": 80, + "remote_identifier": "hthuli" + } +}, +{ + "model": "core.post", + "pk": 3126, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.890Z", + "title": "Make this any sense? Automatic CPU Speed & Power Optimizer", + "body": "
Direct url
", + "author": "spite77", + "publication_date": "2020-07-20T11:53:35Z", + "url": "https://www.reddit.com/r/linux/comments/huikxz/make_this_any_sense_automatic_cpu_speed_power/", + "read": false, + "rule": 80, + "remote_identifier": "huikxz" + } +}, +{ + "model": "core.post", + "pk": 3127, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.891Z", + "title": "Let\u2019s not be pedantic about \u201cOpen Source\u201d", + "body": "
Direct url
", + "author": "speckz", + "publication_date": "2020-07-20T16:46:43Z", + "url": "https://www.reddit.com/r/linux/comments/humirw/lets_not_be_pedantic_about_open_source/", + "read": false, + "rule": 80, + "remote_identifier": "humirw" + } +}, +{ + "model": "core.post", + "pk": 3128, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.893Z", + "title": "Experiences with running Linux Lite", + "body": "
Direct url
", + "author": "daemonpenguin", + "publication_date": "2020-07-20T02:43:49Z", + "url": "https://www.reddit.com/r/linux/comments/hubonw/experiences_with_running_linux_lite/", + "read": false, + "rule": 80, + "remote_identifier": "hubonw" + } +}, +{ + "model": "core.post", + "pk": 3129, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.895Z", + "title": "Tried gnome on arch, surprised how lean it is (used flameshot so it used about 72mb more) closing at 600 megs) on fedora and pop i had gnome eating up 1.3gigs at boot up.", + "body": "
\"Tried
", + "author": "V1n0dKr1shna", + "publication_date": "2020-07-18T13:54:55Z", + "url": "https://www.reddit.com/r/linux/comments/htfeph/tried_gnome_on_arch_surprised_how_lean_it_is_used/", + "read": false, + "rule": 80, + "remote_identifier": "htfeph" + } +}, +{ + "model": "core.post", + "pk": 3130, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.897Z", + "title": "The Free Software Foundation is holding a Fundraiser, help them reach 200 members", + "body": "
Direct url
", + "author": "Neet-Feet", + "publication_date": "2020-07-18T17:55:30Z", + "url": "https://www.reddit.com/r/linux/comments/htiuyi/the_free_software_foundation_is_holding_a/", + "read": false, + "rule": 80, + "remote_identifier": "htiuyi" + } +}, +{ + "model": "core.post", + "pk": 3131, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.899Z", + "title": "Why is the mindset around Arch so negative?", + "body": "

I love the Linux community as a whole. You can find some of the most creative and imaginative people within most Linux communities. On a whole, Linux users are some of the most helpful and informative people you can encounter. Truly the type to think outside the box and learn new things. It can be very inspirational.

\n\n

If I jumped onto Ubuntu, Fedora, or openSUSE's community I can have a free flowing conversation about Linux, their distribution, and getting help or giving help is so free-flowing and easy. The communities are eager to welcome new people and appreciate folks who contribute.

\n\n

Then you have Arch. I love the OS but dislike the mindset. Asking for help is meat with resistance, giving help can also be punishable, and god forbid you try to have a discussion. But it's not just their core community either. For example, I just discovered Endeavour OS which is built around Arch and after 11 post I'm told to come back in 8 hours. Their subReddit here on Reddit, you have to ask to even make 1 post. There of course is also Manjaro Linux and they too have this gatekeeper mindset, the same can be said for ArcoLinux.

\n\n

What is it about Arch that makes everyone want to be either a control freak or a gatekeeper?

\n\n

I do not see this within the Ubuntu or Fedora or openSUSE communities. As I said, their mindset seems eager and willing to unite and work as a community. Am I the only how has noticed this?

\n
", + "author": "Linux-Is-Best", + "publication_date": "2020-07-18T23:28:12Z", + "url": "https://www.reddit.com/r/linux/comments/htojwk/why_is_the_mindset_around_arch_so_negative/", + "read": false, + "rule": 80, + "remote_identifier": "htojwk" + } +}, +{ + "model": "core.post", + "pk": 3132, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.901Z", + "title": "Using the nstat network statistics command in Linux", + "body": "
Direct url
", + "author": "cronos426", + "publication_date": "2020-07-19T17:55:55Z", + "url": "https://www.reddit.com/r/linux/comments/hu2q6v/using_the_nstat_network_statistics_command_in/", + "read": false, + "rule": 80, + "remote_identifier": "hu2q6v" + } +}, +{ + "model": "core.post", + "pk": 3133, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.903Z", + "title": "Contributing via GitLab Merge Requests", + "body": "
Direct url
", + "author": "ChristophCullmann", + "publication_date": "2020-07-18T20:01:26Z", + "url": "https://www.reddit.com/r/linux/comments/htl05p/contributing_via_gitlab_merge_requests/", + "read": false, + "rule": 80, + "remote_identifier": "htl05p" + } +}, +{ + "model": "core.post", + "pk": 3134, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.905Z", + "title": "OpenMandriva: combines WINE64 and 32 into one package capable of running both binaries, i686 architecture was considered as deprecated. Work is underway on a new Rolling release", + "body": "
Direct url
", + "author": "DamonsLinux", + "publication_date": "2020-07-18T15:02:35Z", + "url": "https://www.reddit.com/r/linux/comments/htg9dj/openmandriva_combines_wine64_and_32_into_one/", + "read": false, + "rule": 80, + "remote_identifier": "htg9dj" + } +}, +{ + "model": "core.post", + "pk": 3135, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.906Z", + "title": "OpenRCT2 Player Survey 2020 - Previous survey shows almost 25% players are linux, please help represent linux in the most recent survey", + "body": "
Direct url
", + "author": "christophski", + "publication_date": "2020-07-18T11:39:06Z", + "url": "https://www.reddit.com/r/linux/comments/htdzuh/openrct2_player_survey_2020_previous_survey_shows/", + "read": false, + "rule": 80, + "remote_identifier": "htdzuh" + } +}, +{ + "model": "core.post", + "pk": 3136, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.908Z", + "title": "This week in KDE: Get New Stuff fixes and more", + "body": "
Direct url
", + "author": "kyentei", + "publication_date": "2020-07-18T10:03:46Z", + "url": "https://www.reddit.com/r/linux/comments/htd1an/this_week_in_kde_get_new_stuff_fixes_and_more/", + "read": false, + "rule": 80, + "remote_identifier": "htd1an" + } +}, +{ + "model": "core.post", + "pk": 3137, + "fields": { + "created": "2020-07-20T19:32:35.857Z", + "modified": "2020-07-20T19:32:35.910Z", + "title": "Blender Runs on Linux Pinephone", + "body": "

I managed to get the desktop version of Blender on the Pinephone, and it works really well except for a few bugs.

\n\n

See my post on r/blender:

\n\n

https://www.reddit.com/r/blender/comments/hsxv27/i_installed_blender_on_a_phone/

\n\n

and r/PINE64official:

\n\n

https://www.reddit.com/r/PINE64official/comments/hsxc33/blender_on_pine_phone_almost_usable/

\n\n

I've tried other desktop programs like Xournal and PPSSPP, their UIs also work well, I'd be able to do even more if OpenGL 3 was working.

\n
", + "author": "InfiniteHawk", + "publication_date": "2020-07-17T22:35:14Z", + "url": "https://www.reddit.com/r/linux/comments/ht3d4k/blender_runs_on_linux_pinephone/", + "read": false, + "rule": 80, + "remote_identifier": "ht3d4k" + } +}, +{ + "model": "core.post", + "pk": 3138, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:18:21.616Z", + "title": "Hrmmm They Need to Fix Throttle Animations in the Sabre", + "body": "
", + "author": "TheBootRanger", + "publication_date": "2020-07-21T13:26:01Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv5omc/hrmmm_they_need_to_fix_throttle_animations_in_the/", + "read": true, + "rule": 82, + "remote_identifier": "hv5omc" + } +}, +{ + "model": "core.post", + "pk": 3139, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:18:49.999Z", + "title": "My first 3.10 landing could have gone better...", + "body": "
", + "author": "KnLfey", + "publication_date": "2020-07-21T16:04:50Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv7w85/my_first_310_landing_could_have_gone_better/", + "read": true, + "rule": 82, + "remote_identifier": "hv7w85" + } +}, +{ + "model": "core.post", + "pk": 3140, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:14:50.439Z", + "title": "How about the Christmas in 3 more years?", + "body": "
\"How
", + "author": "SpleanEater", + "publication_date": "2020-07-21T17:49:22Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv9qy8/how_about_the_christmas_in_3_more_years/", + "read": false, + "rule": 82, + "remote_identifier": "hv9qy8" + } +}, +{ + "model": "core.post", + "pk": 3141, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:18:33.532Z", + "title": "Long time Elite Dangerous player. New to star citizen i think im doing great", + "body": "
Direct url
", + "author": "Filblo5", + "publication_date": "2020-07-21T15:33:49Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv7elb/long_time_elite_dangerous_player_new_to_star/", + "read": true, + "rule": 82, + "remote_identifier": "hv7elb" + } +}, +{ + "model": "core.post", + "pk": 3142, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.443Z", + "title": "And we stand by it.", + "body": "
\"And
", + "author": "CyberTill", + "publication_date": "2020-07-21T18:57:48Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvb3wm/and_we_stand_by_it/", + "read": false, + "rule": 82, + "remote_identifier": "hvb3wm" + } +}, +{ + "model": "core.post", + "pk": 3143, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.446Z", + "title": "Nomad", + "body": "
\"Nomad\"
", + "author": "ibracitizen", + "publication_date": "2020-07-21T19:52:24Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvc5h3/nomad/", + "read": false, + "rule": 82, + "remote_identifier": "hvc5h3" + } +}, +{ + "model": "core.post", + "pk": 3144, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.449Z", + "title": "Probably the best screen cap i've ever caught on a whim. 3.5 Arc Corp release. Also a confession: I never pledged. Got a ship with my GPU. I intend to pay my dues.", + "body": "
\"Probably
", + "author": "ScionoicS", + "publication_date": "2020-07-21T20:23:01Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvcqzf/probably_the_best_screen_cap_ive_ever_caught_on_a/", + "read": false, + "rule": 82, + "remote_identifier": "hvcqzf" + } +}, +{ + "model": "core.post", + "pk": 3145, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.451Z", + "title": "Play to escape the depressing job hunt where I need 10 years experience for a entry level job to find this, only been playing for 1 and a half years :(", + "body": "
\"Play
", + "author": "Albert-III-", + "publication_date": "2020-07-21T12:23:45Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv4z08/play_to_escape_the_depressing_job_hunt_where_i/", + "read": false, + "rule": 82, + "remote_identifier": "hv4z08" + } +}, +{ + "model": "core.post", + "pk": 3146, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:19:00.691Z", + "title": "The void beckons.", + "body": "
", + "author": "HisNameWasHis", + "publication_date": "2020-07-21T14:40:51Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv6nij/the_void_beckons/", + "read": true, + "rule": 82, + "remote_identifier": "hv6nij" + } +}, +{ + "model": "core.post", + "pk": 3147, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:19:05.881Z", + "title": "I made a SC-like Photobash with Soldiers", + "body": "
\"I
", + "author": "IsaacPolar", + "publication_date": "2020-07-21T17:13:39Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv92ri/i_made_a_sclike_photobash_with_soldiers/", + "read": true, + "rule": 82, + "remote_identifier": "hv92ri" + } +}, +{ + "model": "core.post", + "pk": 3148, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:19:41.227Z", + "title": "Ocean Shader Improvements", + "body": "
\"Ocean
", + "author": "shoeii", + "publication_date": "2020-07-21T18:41:51Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvasds/ocean_shader_improvements/", + "read": true, + "rule": 82, + "remote_identifier": "hvasds" + } +}, +{ + "model": "core.post", + "pk": 3149, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.459Z", + "title": "As much shit as Star Citizen (rightfully) gets it still does one thing better than any other 'game' I've played", + "body": "

It invokes a real sense of scale, on multiple levels.

\n\n

One could argue that's one of the most important feelings you'd want to capture in any game set in space, but of course it's mostly meaningless if there aren't enough gameplay loops and systems in place to work in tandem with and make the space that's been created interesting, and that's where SC is currently a failure.

\n\n

Even so, I think being able to create that sense of smallness isn't insignificant.

\n\n

You as a pilot are dwarfed by your ship which is itself dwarfed by a larger ship which is itself dwarfed by another, even more massive one which is dwarfed by the space station or hub you're at which is dwarfed by a crater on a moon which is dwarfed by the moon itself which is dwarfed by the planet it orbits which is dwarfed by the sheer vastness of space in between all of those things and that they are, despite the distance, still connected.

\n\n

Getting lost in Lorville (even if it is mostly linear) and knowing it's only a small part of the playable space is a really neat feeling - looking out from the windows of the train up into the sky and knowing you can go there and beyond really makes you feel like there is a whole world (and more) waiting to be explored.

\n\n

I think this is a direct result of having legs and not being locked into the cockpit of your ship - I've played more Elite: Dangerous than Star Citizen and it accomplishes a similar sense of scale but, at least not as far as I've felt, never to the same degree - because you're locked in your ship you never really get this same sense of being small or insignificant even though you are dwarfed in similar ways by planets/asteroids/other ships - will be interesting to see how their implementation of 'space legs' in the upcoming expansion changes this.

\n\n

My favourite thing to do in Star Citizen (because there isn't a whole lot) is to just find some pocket of space far away from anything else and just walk around my ship, feeling truly alone and insignificant, gazing out at the void that stretches infinitely all around - something about that is super comfy.

\n\n

I can't think of many other game that accomplish a similar level of scale though I'm sure they exist.

\n\n

I've been playing an indie game called Empyrion - Galactic Survival and it actually is sort of similar to SC in this regard but it's nowhere near as polished or smooth - transitions from atmosphere to space are not truly seamless and planets themselves are kind of stitched together, but it still manages to invoke that same kind of awe at the scale of things when you dock a small vessel to a capital vessel, for example - definitely worth checking out if you like sci-fi/space games, which you must if you're here, but just be prepared for the jank.

\n
", + "author": "thegreatself", + "publication_date": "2020-07-21T20:30:15Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvcw38/as_much_shit_as_star_citizen_rightfully_gets_it/", + "read": false, + "rule": 82, + "remote_identifier": "hvcw38" + } +}, +{ + "model": "core.post", + "pk": 3150, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.462Z", + "title": "You waiting for patch 3.10 to go live while watching tons of videos about the new flight model features. Be patient, 3.11 and 3.12 will be even better.", + "body": "
\"You
", + "author": "jsabater76", + "publication_date": "2020-07-21T09:39:27Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv372v/you_waiting_for_patch_310_to_go_live_while/", + "read": false, + "rule": 82, + "remote_identifier": "hv372v" + } +}, +{ + "model": "core.post", + "pk": 3151, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.466Z", + "title": "CIG, can we please fix these \"black hole\" doors(when they are closed) on ships please.", + "body": "
\"CIG,
", + "author": "AbnormallyBendPenis", + "publication_date": "2020-07-21T13:40:14Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv5uzj/cig_can_we_please_fix_these_black_hole_doorswhen/", + "read": false, + "rule": 82, + "remote_identifier": "hv5uzj" + } +}, +{ + "model": "core.post", + "pk": 3152, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.468Z", + "title": "Anvil Super Hornet over Cellin", + "body": "
\"Anvil
", + "author": "SaraCaterina", + "publication_date": "2020-07-21T20:33:58Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvcyq6/anvil_super_hornet_over_cellin/", + "read": false, + "rule": 82, + "remote_identifier": "hvcyq6" + } +}, +{ + "model": "core.post", + "pk": 3153, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.471Z", + "title": "3.10 Combat Changes", + "body": "
Direct url
", + "author": "STLYoungblood", + "publication_date": "2020-07-21T16:37:44Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv8fr7/310_combat_changes/", + "read": false, + "rule": 82, + "remote_identifier": "hv8fr7" + } +}, +{ + "model": "core.post", + "pk": 3154, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.472Z", + "title": "Hey CIG how about that S42 Vi.... Oh...", + "body": "
\"Hey
", + "author": "SiEDeN", + "publication_date": "2020-07-21T21:37:16Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hve6am/hey_cig_how_about_that_s42_vi_oh/", + "read": false, + "rule": 82, + "remote_identifier": "hve6am" + } +}, +{ + "model": "core.post", + "pk": 3155, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.475Z", + "title": "3.10 M PTU Eclipse improvements", + "body": "

If this goes live, CIG had addressed 2 of my Eclipse critics.

\n\n

Not because of my videos of course, CIG doesn't know I exist.

\n\n

 

\n\n

a. Eclipse has armor stealth in 3.10, see my table:\nhttps://docs.google.com/spreadsheets/d/1OJXg7MQsG_IVTPsmlmZYaxEPK4n4iqnhQx4oigIlJHg/edit#gid=343807746

\n\n

 

\n\n

b. Eclipse can fire her size 9 torpedoes way quicker now, see my video with a side by side comparison of the max firing speed in 3.9 and 3.10:\nhttps://youtu.be/GFTF1Qt7T3o?t=207

\n
", + "author": "Camural", + "publication_date": "2020-07-21T18:15:50Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hva9lc/310_m_ptu_eclipse_improvements/", + "read": false, + "rule": 82, + "remote_identifier": "hva9lc" + } +}, +{ + "model": "core.post", + "pk": 3156, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.477Z", + "title": "Hark! The Drake Herald Sings", + "body": "
\"Hark!
", + "author": "CyrexStorm", + "publication_date": "2020-07-21T16:19:31Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv84kk/hark_the_drake_herald_sings/", + "read": false, + "rule": 82, + "remote_identifier": "hv84kk" + } +}, +{ + "model": "core.post", + "pk": 3157, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.479Z", + "title": "The new flight stick in the Prowler", + "body": "
\"The
", + "author": "Potato_Nades", + "publication_date": "2020-07-21T16:22:22Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv86c2/the_new_flight_stick_in_the_prowler/", + "read": false, + "rule": 82, + "remote_identifier": "hv86c2" + } +}, +{ + "model": "core.post", + "pk": 3158, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.481Z", + "title": "Norwegian VAT charged from August 1st", + "body": "
\"Norwegian
", + "author": "norgeek", + "publication_date": "2020-07-21T10:30:57Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv3r3l/norwegian_vat_charged_from_august_1st/", + "read": false, + "rule": 82, + "remote_identifier": "hv3r3l" + } +}, +{ + "model": "core.post", + "pk": 3159, + "fields": { + "created": "2020-07-21T20:14:50.423Z", + "modified": "2020-07-21T20:14:50.484Z", + "title": "With Pyro (currently WIP), Nyx (partially done), Odin (S42), currently on the way, what is everyone\u2019s thoughts on Terra possibly being next on the list of star systems to be added into the PU within \u2026", + "body": "
\"With
", + "author": "realCLTotaku", + "publication_date": "2020-07-21T13:27:09Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv5p41/with_pyro_currently_wip_nyx_partially_done_odin/", + "read": false, + "rule": 82, + "remote_identifier": "hv5p41" + } +}, +{ + "model": "core.post", + "pk": 3160, + "fields": { + "created": "2020-07-21T20:14:50.423Z", + "modified": "2020-07-21T20:14:50.486Z", + "title": "Testing out the new electron rifle", + "body": "
", + "author": "joshbaker2112", + "publication_date": "2020-07-21T02:56:19Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huxr6d/testing_out_the_new_electron_rifle/", + "read": false, + "rule": 82, + "remote_identifier": "huxr6d" + } +}, +{ + "model": "core.post", + "pk": 3161, + "fields": { + "created": "2020-07-21T20:14:50.423Z", + "modified": "2020-07-21T20:14:50.487Z", + "title": "Imperial Geographic's Lovecraftian magazine special is here. \ud83d\udc19 Find the link in the comments!", + "body": "
\"Imperial
", + "author": "Good_Punk2", + "publication_date": "2020-07-21T18:21:38Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvadrh/imperial_geographics_lovecraftian_magazine/", + "read": false, + "rule": 82, + "remote_identifier": "hvadrh" + } +}, +{ + "model": "core.post", + "pk": 3162, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.525Z", + "title": "Linux Distributions Timeline", + "body": "
\"Linux
", + "author": "bauripalash", + "publication_date": "2020-07-21T06:07:59Z", + "url": "https://www.reddit.com/r/linux/comments/hv0ktn/linux_distributions_timeline/", + "read": false, + "rule": 80, + "remote_identifier": "hv0ktn" + } +}, +{ + "model": "core.post", + "pk": 3163, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.527Z", + "title": "Fedora: Proposal to replace default wined3d backend with DXVK", + "body": "
Direct url
", + "author": "friskfrugt", + "publication_date": "2020-07-21T19:42:49Z", + "url": "https://www.reddit.com/r/linux/comments/hvbyyr/fedora_proposal_to_replace_default_wined3d/", + "read": false, + "rule": 80, + "remote_identifier": "hvbyyr" + } +}, +{ + "model": "core.post", + "pk": 3164, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.531Z", + "title": "Update on marketing and communication plans for the LibreOffice 7.x series", + "body": "
Direct url
", + "author": "TheQuantumZero", + "publication_date": "2020-07-21T09:59:23Z", + "url": "https://www.reddit.com/r/linux/comments/hv3erm/update_on_marketing_and_communication_plans_for/", + "read": false, + "rule": 80, + "remote_identifier": "hv3erm" + } +}, +{ + "model": "core.post", + "pk": 3165, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.533Z", + "title": "FOSS job opening: LibreOffice Development Mentor at The Document Foundation", + "body": "
Direct url
", + "author": "themikeosguy", + "publication_date": "2020-07-21T14:26:36Z", + "url": "https://www.reddit.com/r/linux/comments/hv6gfw/foss_job_opening_libreoffice_development_mentor/", + "read": false, + "rule": 80, + "remote_identifier": "hv6gfw" + } +}, +{ + "model": "core.post", + "pk": 3166, + "fields": { + "created": "2020-07-21T20:14:50.503Z", + "modified": "2020-07-21T20:14:50.536Z", + "title": "gomd - quickly display formatted markdown files with code highlight in your browser", + "body": "

Hi all!

\n\n

I wanted to share a project I've been working on recently. I think it reached a stage where it's pretty usable and should work out of the box. gomd sets up a HTTP server and serves a directory in your browser so you can quickly view your markdown files. It comes with some neat features like:

\n\n\n\n

Link: gomd

\n\n

For now its only available from AUR or built from source.

\n\n

\n\n

Any tips or feedback will be greatly appreciated :)

\n
", + "author": "wwojtekk", + "publication_date": "2020-07-21T20:07:31Z", + "url": "https://www.reddit.com/r/linux/comments/hvcg44/gomd_quickly_display_formatted_markdown_files/", + "read": false, + "rule": 80, + "remote_identifier": "hvcg44" + } +}, +{ + "model": "core.post", + "pk": 3167, + "fields": { + "created": "2020-07-21T20:14:50.503Z", + "modified": "2020-07-21T20:14:50.543Z", + "title": "They're not otherwise wrong, but it didn't become a real Internet standard until 2017.", + "body": "
\"They're
", + "author": "foodown", + "publication_date": "2020-07-21T21:39:09Z", + "url": "https://www.reddit.com/r/linux/comments/hve7l5/theyre_not_otherwise_wrong_but_it_didnt_become_a/", + "read": false, + "rule": 80, + "remote_identifier": "hve7l5" + } +}, +{ + "model": "core.post", + "pk": 3168, + "fields": { + "created": "2020-07-21T20:14:50.503Z", + "modified": "2020-07-21T20:14:50.545Z", + "title": "Drawing - an alternative to Paint for Linux (gtk3, support HiDPI)", + "body": "
Direct url
", + "author": "dontdieych", + "publication_date": "2020-07-21T02:37:22Z", + "url": "https://www.reddit.com/r/linux/comments/huxgsg/drawing_an_alternative_to_paint_for_linux_gtk3/", + "read": false, + "rule": 80, + "remote_identifier": "huxgsg" + } +}, +{ + "model": "core.post", + "pk": 3169, + "fields": { + "created": "2020-07-21T20:14:50.509Z", + "modified": "2020-07-21T20:14:50.547Z", + "title": "Observations on a Linux issue with 3.5mm earphones with a mic", + "body": "

Alright hello. I have come from r/SolusProject and I made a post there to do with headphone issues. I suggest you read through the post and comments to get a better understanding before reading this https://www.reddit.com/r/SolusProject/comments/hsql4d/frustrating_headphone_issues/. I had posted to do with it again, but it got taken down for duplication (when it wasn't duplication). This post is more of my observations from experimenting and such. There are distros I haven't tried but I tried a wide range of distros like manjaro, ubuntu based ones and all solus flavors, and I was looking more for how well they worked out of the box, rather than with fiddling around with pulse, hdajack etc which I know will work eventually. If you stumbled across this from searching about the same issue I have (or similar) or are confused to what this is about, I suggest you look at my previous post also.

\n\n

So anyways, I've tried the past few days mounting isos to usb drives and trying live os and installing various distros to see about the headphone issue. And my conclusion is that this issue affects the linux kernel in some way across the board. I don't really understand why completely but I have some kind of idea.

\n\n

From installing fresh distros, I noticed that the earphones (they are 3.5mm earphones + mic) get recognised as a microphone and not as a speaker system of some kind. Every single time I had a look at the sound settings and in pulse, they came up as plugged microphone, with the internal speakers being the only output device every single time. It's really odd seeing as how ubuntu 14.04 and xubuntu etc from years past worked flawlessly with the earphones, even manjaro a while ago on my older craptop worked fine. I don't really understand why it doesn't work on my device now.

\n\n

I'll leave my specs at the bottom of this post but what I think is is there's something the manufacturer did, or something like the cpu causes issue with linux. The manufacturer of my laptop is Lenovo, and the cpu/igpu is from AMD. A warning sign is that when installing a linux distro, it doesn't bring up the dual boot menu at startup like it should. Instead it completely hides the fact it exists until I use something like easyuefi to add an option for that distro, how it works is you specify the boot partition, whether it's linux or windows and the loader conf file for the distro. All of this hassle everytime doesn't appear on my craptop, because the dual boot menu appears flawlessly without issue. May be because it uses an Intel cpu/igpu unlike my newer laptop but it's hard to say.

\n\n

Also, it seems like the devices that appear in a given distro when looking at alsa, is hd generic devices but by reloading alsa or any command that shows the full name of the device, it says it's Intel. I don't know if that would be an issue, maybe amd use intel sound drivers or something. It's odd nonetheless.

\n\n

This issue has been boggling my mind for obvious reasons, with half-rhetorical questions like does linux not support the earphones anymore, whether out of accident from an overlooked bug in an update or intentionally phasing out? Is any of this AMD or Lenovo's fault? Even with proper headphones or something, will they fail? I don't think anyone here really knows, hell I'd bet an extreme that no one really understands why in the linux community. I kinda rambled in this post with stuff that should've been said in the last post/thread, but I'm saying it now.

\n\n

Thanks for contributing thus far to this discussion in figuring this out.

\n\n

Specs: AMD Ryzen 5 3500U Mobile CPU (2.2 - 3.7ghz quad core)

\n\n

Radeon Vega 8 Integrated GPU, 8GB Ram, 256GB SSD.

\n\n

Lenovo C340-14API Laptop

\n
", + "author": "BrianMeerkatlol", + "publication_date": "2020-07-21T21:02:19Z", + "url": "https://www.reddit.com/r/linux/comments/hvdi3o/observations_on_a_linux_issue_with_35mm_earphones/", + "read": false, + "rule": 80, + "remote_identifier": "hvdi3o" + } +}, +{ + "model": "core.post", + "pk": 3170, + "fields": { + "created": "2020-07-21T20:14:50.509Z", + "modified": "2020-07-21T20:14:50.549Z", + "title": "South Korean distro HamoniKR OS has been added to Distrowatch", + "body": "
Direct url
", + "author": "TheHordeRisesAgain", + "publication_date": "2020-07-21T07:44:21Z", + "url": "https://www.reddit.com/r/linux/comments/hv1ug1/south_korean_distro_hamonikr_os_has_been_added_to/", + "read": false, + "rule": 80, + "remote_identifier": "hv1ug1" + } +}, +{ + "model": "core.post", + "pk": 3171, + "fields": { + "created": "2020-07-21T20:14:50.509Z", + "modified": "2020-07-21T20:14:50.559Z", + "title": "The Jailer is free! New release of the outstanding database subsetter and browser is available.", + "body": "
Direct url
", + "author": "Plane-Discussion", + "publication_date": "2020-07-21T12:53:54Z", + "url": "https://www.reddit.com/r/linux/comments/hv5b0j/the_jailer_is_free_new_release_of_the_outstanding/", + "read": false, + "rule": 80, + "remote_identifier": "hv5b0j" + } +}, +{ + "model": "core.post", + "pk": 3172, + "fields": { + "created": "2020-07-21T20:14:50.513Z", + "modified": "2020-07-21T20:14:50.563Z", + "title": "A few very well-aged excerpts from Microsoft\u2019s infamous 2004 \u201cGet the facts\u201d campaign, where they make the case for Windows servers being cheaper, more secure, and more performant than Linux servers", + "body": "
\n

Get the facts on Windows and Linux.

\n\n

Leading companies and third-party analysts confirm it: Windows has a lower total cost of ownership and outperforms Linux.

\n\n

...

\n\n

-Security

\n\n

Windows Users Have Fewer Vulnerabilities

\n
\n\n

And then literally the very next bullet point:

\n\n
\n

-Featured Customer Case Study

\n\n

Equifax

\n\n

Equifax Sees 14 Percent Cost Savings

\n\n

Find out why Equifax, a global leader in transforming data into intelligence, selected Windows over Linux to enhance the speed and performance of its marketing services capabilities. Using Microsoft Windows Server System, the company has seen 14 percent in cost savings over Linux.

\n
\n\n

Good thing they saved 14% and got all that extra security! Sure their website is janky and their login flow is downright horrifying (Check it out if you want to be amazed), but who could blame them? Linux is \u201cProhibitively Expensive, Extremely Complex, and Provides No Tangible Business Gains\u201d, Microsoft said so!

\n\n

Source: https://web.archive.org/web/20041027003759/http://www.microsoft.com/windowsserversystem/facts/default.mspx

\n
", + "author": "kevinhaze", + "publication_date": "2020-07-20T21:42:15Z", + "url": "https://www.reddit.com/r/linux/comments/hus5lz/a_few_very_wellaged_excerpts_from_microsofts/", + "read": false, + "rule": 80, + "remote_identifier": "hus5lz" + } +}, +{ + "model": "core.post", + "pk": 3173, + "fields": { + "created": "2020-07-21T20:14:50.515Z", + "modified": "2020-07-21T20:14:50.566Z", + "title": "Are there are any professional audio recording studios or artists that use Linux?", + "body": "

As the title says, who is using Linux as a professional audio engineer, producer, or artist? I am a former Mac user myself, and I am seeing people from time to time who have become disillusioned with what Apple has been doing for the past few years. However, I'm not sure if Linux really has a place for these people to land if they are serious about what they do.

\n\n

Fedora Design Suite and Ubuntu Studio are definitely encouraging to see, but what is their real-world usage like? Are we getting better with professional audio in Linux, or have things been stagnant for years?

\n
", + "author": "RootHouston", + "publication_date": "2020-07-21T00:08:26Z", + "url": "https://www.reddit.com/r/linux/comments/huuxvq/are_there_are_any_professional_audio_recording/", + "read": false, + "rule": 80, + "remote_identifier": "huuxvq" + } +}, +{ + "model": "core.post", + "pk": 3174, + "fields": { + "created": "2020-07-21T20:14:50.515Z", + "modified": "2020-07-21T20:14:50.570Z", + "title": "When Linux had marketing", + "body": "
Direct url
", + "author": "Commodore256", + "publication_date": "2020-07-21T14:03:56Z", + "url": "https://www.reddit.com/r/linux/comments/hv65oa/when_linux_had_marketing/", + "read": false, + "rule": 80, + "remote_identifier": "hv65oa" + } +}, +{ + "model": "core.post", + "pk": 3175, + "fields": { + "created": "2020-07-21T20:14:50.520Z", + "modified": "2020-07-21T20:14:50.598Z", + "title": "Ward: Simple and minimalistic server dashboard", + "body": "

Ward is a simple and and minimalistic server monitoring tool. Ward supports adaptive design system. Also it supports dark theme. It shows only principal information and can be used, if you want to see nice looking dashboard instead looking on bunch of numbers and graphs. Ward works nice on all popular operating systems, because it uses OSHI.

\n\n

https://preview.redd.it/gdppswc3a3c51.png?width=1448&format=png&auto=webp&s=0d6e10146c105ddcfd045dd59c970d4c127ddb8c

\n\n

https://github.com/B-Software/Ward

\n
", + "author": "Pabyzu", + "publication_date": "2020-07-21T00:33:40Z", + "url": "https://www.reddit.com/r/linux/comments/huvea3/ward_simple_and_minimalistic_server_dashboard/", + "read": false, + "rule": 80, + "remote_identifier": "huvea3" + } +}, +{ + "model": "core.post", + "pk": 3176, + "fields": { + "created": "2020-07-21T20:14:50.522Z", + "modified": "2020-07-21T20:14:50.606Z", + "title": "WindowsFX - a good Windows alternative?", + "body": "

I would personally like to hear some of your opinions (in the replies) about WindowsFX. What is WindowsFX you may ask? WindowsFX is a Brazilian linux distribution that is designed to look and act like Windows 10.

\n\n

Linux / WindowsFX is based off of Ubuntu, and uses Cinnamon as its DE. Upon first boot, normal Windows users can tell the difference. But if you were to put it in front of a non tech-savvy person, they wouldn't be able to tell the difference.

\n\n

Personally, with WSL on Windows, I see no need for a distro like this. However, as I said, I would like to hear your opinions on this distro.

\n\n

Video review here.

\n
", + "author": "Demonitized101", + "publication_date": "2020-07-20T23:03:29Z", + "url": "https://www.reddit.com/r/linux/comments/hutpt5/windowsfx_a_good_windows_alternative/", + "read": false, + "rule": 80, + "remote_identifier": "hutpt5" + } +}, +{ + "model": "core.post", + "pk": 3177, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:14:50.780Z", + "title": "Every day this good boy brings a carrot to his best buddy", + "body": "
", + "author": "TooShiftyForYou", + "publication_date": "2020-07-21T15:25:31Z", + "url": "https://www.reddit.com/r/aww/comments/hv7a8b/every_day_this_good_boy_brings_a_carrot_to_his/", + "read": false, + "rule": 81, + "remote_identifier": "hv7a8b" + } +}, +{ + "model": "core.post", + "pk": 3178, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-25T20:08:34.264Z", + "title": "Kitten mimics his human petting the dog", + "body": "
", + "author": "SpecterAscendant", + "publication_date": "2020-07-21T14:56:57Z", + "url": "https://www.reddit.com/r/aww/comments/hv6ve3/kitten_mimics_his_human_petting_the_dog/", + "read": true, + "rule": 81, + "remote_identifier": "hv6ve3" + } +}, +{ + "model": "core.post", + "pk": 3179, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:14:50.789Z", + "title": "My fox friend!", + "body": "
", + "author": "Zepantha", + "publication_date": "2020-07-21T14:27:25Z", + "url": "https://www.reddit.com/r/aww/comments/hv6gte/my_fox_friend/", + "read": false, + "rule": 81, + "remote_identifier": "hv6gte" + } +}, +{ + "model": "core.post", + "pk": 3180, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:15:46.876Z", + "title": "Ducks annihilate peas", + "body": "
", + "author": "tommycalibre", + "publication_date": "2020-07-21T17:12:40Z", + "url": "https://www.reddit.com/r/aww/comments/hv9258/ducks_annihilate_peas/", + "read": true, + "rule": 81, + "remote_identifier": "hv9258" + } +}, +{ + "model": "core.post", + "pk": 3181, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:14:50.797Z", + "title": "Wiggle it baby", + "body": "
", + "author": "neo_star", + "publication_date": "2020-07-21T18:44:31Z", + "url": "https://www.reddit.com/r/aww/comments/hvaucy/wiggle_it_baby/", + "read": false, + "rule": 81, + "remote_identifier": "hvaucy" + } +}, +{ + "model": "core.post", + "pk": 3182, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:16:22.725Z", + "title": "I guess I should do this.. everyone seems to be liking little pups and kittens so.. Reddit, meet bailey", + "body": "
\"I
", + "author": "X_XNOTHINGX_X", + "publication_date": "2020-07-21T14:15:08Z", + "url": "https://www.reddit.com/r/aww/comments/hv6b0a/i_guess_i_should_do_this_everyone_seems_to_be/", + "read": true, + "rule": 81, + "remote_identifier": "hv6b0a" + } +}, +{ + "model": "core.post", + "pk": 3183, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.806Z", + "title": "The hat makes the crab.", + "body": "
\"The
", + "author": "fujfuj", + "publication_date": "2020-07-21T14:48:40Z", + "url": "https://www.reddit.com/r/aww/comments/hv6rde/the_hat_makes_the_crab/", + "read": false, + "rule": 81, + "remote_identifier": "hv6rde" + } +}, +{ + "model": "core.post", + "pk": 3184, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.812Z", + "title": "Baby bunny fits in hand", + "body": "
", + "author": "Hawken10", + "publication_date": "2020-07-21T12:31:30Z", + "url": "https://www.reddit.com/r/aww/comments/hv5253/baby_bunny_fits_in_hand/", + "read": false, + "rule": 81, + "remote_identifier": "hv5253" + } +}, +{ + "model": "core.post", + "pk": 3185, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.818Z", + "title": "My cat and I, both pregnant", + "body": "
\"My
", + "author": "nixdionisio", + "publication_date": "2020-07-21T11:06:25Z", + "url": "https://www.reddit.com/r/aww/comments/hv44m2/my_cat_and_i_both_pregnant/", + "read": false, + "rule": 81, + "remote_identifier": "hv44m2" + } +}, +{ + "model": "core.post", + "pk": 3186, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.822Z", + "title": "Very sweet dance", + "body": "
", + "author": "Ashley1023", + "publication_date": "2020-07-21T13:03:03Z", + "url": "https://www.reddit.com/r/aww/comments/hv5ewq/very_sweet_dance/", + "read": false, + "rule": 81, + "remote_identifier": "hv5ewq" + } +}, +{ + "model": "core.post", + "pk": 3187, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.825Z", + "title": "My local pet-store has a cat named Vegemite \u2764\ufe0f", + "body": "
\"My
", + "author": "galinhad", + "publication_date": "2020-07-21T12:06:17Z", + "url": "https://www.reddit.com/r/aww/comments/hv4s5z/my_local_petstore_has_a_cat_named_vegemite/", + "read": false, + "rule": 81, + "remote_identifier": "hv4s5z" + } +}, +{ + "model": "core.post", + "pk": 3188, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-21T20:15:01.459Z", + "title": "A teacher like that makes a huge difference", + "body": "
", + "author": "Unicornglitteryblood", + "publication_date": "2020-07-21T18:29:57Z", + "url": "https://www.reddit.com/r/aww/comments/hvajo9/a_teacher_like_that_makes_a_huge_difference/", + "read": true, + "rule": 81, + "remote_identifier": "hvajo9" + } +}, +{ + "model": "core.post", + "pk": 3189, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-22T19:55:49.930Z", + "title": "Kitten Encounters Bubbly Water", + "body": "
\"Kitten
", + "author": "DragonOBunny", + "publication_date": "2020-07-21T15:28:05Z", + "url": "https://www.reddit.com/r/aww/comments/hv7bis/kitten_encounters_bubbly_water/", + "read": true, + "rule": 81, + "remote_identifier": "hv7bis" + } +}, +{ + "model": "core.post", + "pk": 3190, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-21T20:14:50.833Z", + "title": "Are These My Chickens Now?", + "body": "
Direct url
", + "author": "jasontaken", + "publication_date": "2020-07-21T09:55:36Z", + "url": "https://www.reddit.com/r/aww/comments/hv3de1/are_these_my_chickens_now/", + "read": false, + "rule": 81, + "remote_identifier": "hv3de1" + } +}, +{ + "model": "core.post", + "pk": 3191, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-25T20:08:20.518Z", + "title": "Our St Bernard 6 months apart", + "body": "
\"Our
", + "author": "ryan3105", + "publication_date": "2020-07-21T18:00:04Z", + "url": "https://www.reddit.com/r/aww/comments/hv9yea/our_st_bernard_6_months_apart/", + "read": true, + "rule": 81, + "remote_identifier": "hv9yea" + } +}, +{ + "model": "core.post", + "pk": 3192, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-21T20:14:50.837Z", + "title": "Father and child in sync", + "body": "
", + "author": "Araragi_Monogatari", + "publication_date": "2020-07-21T08:29:18Z", + "url": "https://www.reddit.com/r/aww/comments/hv2enj/father_and_child_in_sync/", + "read": false, + "rule": 81, + "remote_identifier": "hv2enj" + } +}, +{ + "model": "core.post", + "pk": 3193, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.840Z", + "title": "A meme is born", + "body": "
\"A
", + "author": "Unicornglitteryblood", + "publication_date": "2020-07-21T18:55:04Z", + "url": "https://www.reddit.com/r/aww/comments/hvb1vh/a_meme_is_born/", + "read": false, + "rule": 81, + "remote_identifier": "hvb1vh" + } +}, +{ + "model": "core.post", + "pk": 3194, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.842Z", + "title": "She bites, then she sleeps, then bites again, then sleeps again. \ud83d\ude02", + "body": "
", + "author": "earlymauvs", + "publication_date": "2020-07-21T11:34:19Z", + "url": "https://www.reddit.com/r/aww/comments/hv4fat/she_bites_then_she_sleeps_then_bites_again_then/", + "read": false, + "rule": 81, + "remote_identifier": "hv4fat" + } +}, +{ + "model": "core.post", + "pk": 3195, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.844Z", + "title": "Nothing calmer that 2 ginger cats rubbing heads and showing their love in morning", + "body": "
\"Nothing
", + "author": "Apotheosis33", + "publication_date": "2020-07-21T08:39:24Z", + "url": "https://www.reddit.com/r/aww/comments/hv2j2g/nothing_calmer_that_2_ginger_cats_rubbing_heads/", + "read": false, + "rule": 81, + "remote_identifier": "hv2j2g" + } +}, +{ + "model": "core.post", + "pk": 3196, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.851Z", + "title": "Ring Tailed Possum", + "body": "
Direct url
", + "author": "Wayward-Delver", + "publication_date": "2020-07-21T11:23:51Z", + "url": "https://www.reddit.com/r/aww/comments/hv4b9e/ring_tailed_possum/", + "read": false, + "rule": 81, + "remote_identifier": "hv4b9e" + } +}, +{ + "model": "core.post", + "pk": 3197, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.854Z", + "title": "Baby scooby in sad mood....", + "body": "
\"Baby
", + "author": "deepanshuahiroo7", + "publication_date": "2020-07-21T15:12:23Z", + "url": "https://www.reddit.com/r/aww/comments/hv73ft/baby_scooby_in_sad_mood/", + "read": false, + "rule": 81, + "remote_identifier": "hv73ft" + } +}, +{ + "model": "core.post", + "pk": 3198, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:14:50.856Z", + "title": "New friends!", + "body": "
\"New
", + "author": "HelentotheKeller", + "publication_date": "2020-07-21T13:10:48Z", + "url": "https://www.reddit.com/r/aww/comments/hv5i6i/new_friends/", + "read": false, + "rule": 81, + "remote_identifier": "hv5i6i" + } +}, +{ + "model": "core.post", + "pk": 3199, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:14:50.858Z", + "title": "When you haven't chewed anything for 1 second", + "body": "
\"When
", + "author": "Tanay4", + "publication_date": "2020-07-21T10:26:53Z", + "url": "https://www.reddit.com/r/aww/comments/hv3pl0/when_you_havent_chewed_anything_for_1_second/", + "read": false, + "rule": 81, + "remote_identifier": "hv3pl0" + } +}, +{ + "model": "core.post", + "pk": 3200, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:17:01.490Z", + "title": "Mango Derp", + "body": "
\"Mango
", + "author": "sheetglass", + "publication_date": "2020-07-21T13:27:26Z", + "url": "https://www.reddit.com/r/aww/comments/hv5p8s/mango_derp/", + "read": true, + "rule": 81, + "remote_identifier": "hv5p8s" + } +}, +{ + "model": "core.post", + "pk": 3201, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:14:50.863Z", + "title": "My guy turns 20 next month", + "body": "
\"My
", + "author": "alozsoc", + "publication_date": "2020-07-21T06:34:26Z", + "url": "https://www.reddit.com/r/aww/comments/hv0xp1/my_guy_turns_20_next_month/", + "read": false, + "rule": 81, + "remote_identifier": "hv0xp1" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "add_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "change_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "delete_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "view_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "add_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "change_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "delete_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "view_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add group", + "content_type": [ + "auth", + "group" + ], + "codename": "add_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change group", + "content_type": [ + "auth", + "group" + ], + "codename": "change_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete group", + "content_type": [ + "auth", + "group" + ], + "codename": "delete_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view group", + "content_type": [ + "auth", + "group" + ], + "codename": "view_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "add_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "change_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "delete_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "view_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add session", + "content_type": [ + "sessions", + "session" + ], + "codename": "add_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change session", + "content_type": [ + "sessions", + "session" + ], + "codename": "change_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete session", + "content_type": [ + "sessions", + "session" + ], + "codename": "delete_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view session", + "content_type": [ + "sessions", + "session" + ], + "codename": "view_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "add_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "change_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "delete_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "view_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "add_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "change_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "delete_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "view_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "add_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "change_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "delete_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "view_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "add_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "change_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "delete_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "view_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "add_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "change_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "delete_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "view_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "add_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "change_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "delete_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "view_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "add_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "change_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "delete_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "view_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "add_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "change_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "delete_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "view_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "add_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "change_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "delete_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "view_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "add_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "change_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "delete_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "view_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add user", + "content_type": [ + "accounts", + "user" + ], + "codename": "add_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change user", + "content_type": [ + "accounts", + "user" + ], + "codename": "change_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete user", + "content_type": [ + "accounts", + "user" + ], + "codename": "delete_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view user", + "content_type": [ + "accounts", + "user" + ], + "codename": "view_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add post", + "content_type": [ + "core", + "post" + ], + "codename": "add_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change post", + "content_type": [ + "core", + "post" + ], + "codename": "change_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete post", + "content_type": [ + "core", + "post" + ], + "codename": "delete_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view post", + "content_type": [ + "core", + "post" + ], + "codename": "view_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add Category", + "content_type": [ + "core", + "category" + ], + "codename": "add_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change Category", + "content_type": [ + "core", + "category" + ], + "codename": "change_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete Category", + "content_type": [ + "core", + "category" + ], + "codename": "delete_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view Category", + "content_type": [ + "core", + "category" + ], + "codename": "view_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "add_collectionrule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "change_collectionrule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "delete_collectionrule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "view_collectionrule" + } +}, +{ + "model": "accounts.user", + "fields": { + "password": "pbkdf2_sha256$180000$U9a2CS9X0b8Y$T6bD/VoUOFoGNIp16aFlOL0N7q0e6A3I97ypm/AhsGo=", + "last_login": "2020-07-21T20:14:35.966Z", + "is_superuser": true, + "first_name": "", + "last_name": "", + "is_staff": true, + "is_active": true, + "date_joined": "2019-07-18T18:52:36.080Z", + "email": "sonny@bakker.nl", + "task": 10, + "reddit_refresh_token": null, + "reddit_access_token": null, + "groups": [], + "user_permissions": [] + } +}, +{ + "model": "core.category", + "pk": 8, + "fields": { + "created": "2019-11-17T19:37:24.671Z", + "modified": "2019-11-18T19:59:55.010Z", + "name": "World news", + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "core.category", + "pk": 9, + "fields": { + "created": "2019-11-17T19:37:26.161Z", + "modified": "2020-05-30T13:36:10.509Z", + "name": "Tech", + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 3, + "fields": { + "created": "2019-07-14T13:08:10.374Z", + "modified": "2020-07-14T11:45:30.680Z", + "name": "Hackers News", + "type": "feed", + "url": "https://news.ycombinator.com/rss", + "website_url": "https://news.ycombinator.com/", + "favicon": "https://news.ycombinator.com/favicon.ico", + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:30.477Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 4, + "fields": { + "created": "2019-07-20T11:24:32.745Z", + "modified": "2020-07-14T11:45:29.357Z", + "name": "BBC", + "type": "feed", + "url": "http://feeds.bbci.co.uk/news/world/rss.xml", + "website_url": "https://www.bbc.co.uk/news/", + "favicon": "https://m.files.bbci.co.uk/modules/bbc-morph-news-waf-page-meta/2.5.2/apple-touch-icon-57x57-precomposed.png", + "timezone": "UTC", + "category": 8, + "last_run": "2020-07-14T11:45:28.863Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 5, + "fields": { + "created": "2019-07-20T11:24:50.411Z", + "modified": "2020-07-14T11:45:30.063Z", + "name": "Ars Technica", + "type": "feed", + "url": "http://feeds.arstechnica.com/arstechnica/index?fmt=xml", + "website_url": "https://arstechnica.com", + "favicon": "https://cdn.arstechnica.net/favicon.ico", + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:29.810Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 6, + "fields": { + "created": "2019-07-20T11:25:02.089Z", + "modified": "2020-07-14T11:45:30.473Z", + "name": "The Guardian", + "type": "feed", + "url": "https://www.theguardian.com/world/rss", + "website_url": "https://www.theguardian.com/world", + "favicon": "https://assets.guim.co.uk/images/favicons/873381bf11d58e20f551905d51575117/72x72.png", + "timezone": "UTC", + "category": 8, + "last_run": "2020-07-14T11:45:30.181Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 7, + "fields": { + "created": "2019-07-20T11:25:30.121Z", + "modified": "2020-07-14T11:45:29.807Z", + "name": "Tweakers", + "type": "feed", + "url": "http://feeds.feedburner.com/tweakers/mixed?fmt=xml", + "website_url": "https://tweakers.net/", + "favicon": null, + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:29.525Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 8, + "fields": { + "created": "2019-07-20T11:25:46.256Z", + "modified": "2020-07-14T11:45:30.179Z", + "name": "The Verge", + "type": "feed", + "url": "https://www.theverge.com/rss/index.xml", + "website_url": "https://www.theverge.com/", + "favicon": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395367/favicon-16x16.0.png", + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:30.066Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 9, + "fields": { + "created": "2019-11-24T15:28:41.399Z", + "modified": "2020-07-14T11:45:29.522Z", + "name": "NOS", + "type": "feed", + "url": "http://feeds.nos.nl/nosnieuwsalgemeen", + "website_url": null, + "favicon": null, + "timezone": "Europe/Amsterdam", + "category": 8, + "last_run": "2020-07-14T11:45:29.362Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 80, + "fields": { + "created": "2020-07-08T19:30:10.638Z", + "modified": "2020-07-21T20:14:50.609Z", + "name": "Linux subreddit", + "type": "subreddit", + "url": "https://oauth.reddit.com/r/linux/hot", + "website_url": null, + "favicon": null, + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-21T20:14:50.492Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 81, + "fields": { + "created": "2020-07-08T19:30:33.590Z", + "modified": "2020-07-21T20:14:50.865Z", + "name": "AWW subreddit", + "type": "subreddit", + "url": "https://oauth.reddit.com/r/aww/hot", + "website_url": null, + "favicon": null, + "timezone": "UTC", + "category": 8, + "last_run": "2020-07-21T20:14:50.768Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 82, + "fields": { + "created": "2020-07-20T19:29:37.675Z", + "modified": "2020-07-21T20:14:50.489Z", + "name": "Star citizen subreddit", + "type": "subreddit", + "url": "https://oauth.reddit.com/r/starcitizen/hot.json", + "website_url": null, + "favicon": null, + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-21T20:14:50.355Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "admin.logentry", + "pk": 1, + "fields": { + "action_time": "2020-05-24T18:38:44.624Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "object_id": "5", + "object_repr": "every 4 hours", + "action_flag": 1, + "change_message": "[{\"added\": {}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 2, + "fields": { + "action_time": "2020-05-24T18:38:46.689Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "10", + "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Interval Schedule\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 3, + "fields": { + "action_time": "2020-05-24T18:39:09.203Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "26", + "object_repr": "sonnyba871@gmail.com-collection-task: every hour", + "action_flag": 3, + "change_message": "" + } +}, +{ + "model": "admin.logentry", + "pk": 4, + "fields": { + "action_time": "2020-05-24T19:46:50.248Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "10", + "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Positional Arguments\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 5, + "fields": { + "action_time": "2020-07-07T19:37:57.086Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Reddit refresh token\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 6, + "fields": { + "action_time": "2020-07-07T19:39:46.160Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "10", + "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Task (registered)\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 7, + "fields": { + "action_time": "2020-07-08T19:29:27.025Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "11", + "object_repr": "Reddit collection task: every 4 hours", + "action_flag": 1, + "change_message": "[{\"added\": {}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 8, + "fields": { + "action_time": "2020-07-14T11:46:50.039Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 9, + "fields": { + "action_time": "2020-07-18T19:08:33.997Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "collection", + "collectionrule" + ], + "object_id": "81", + "object_repr": "AWW subreddit", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 10, + "fields": { + "action_time": "2020-07-18T19:08:44.063Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "collection", + "collectionrule" + ], + "object_id": "80", + "object_repr": "Linux subreddit", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 11, + "fields": { + "action_time": "2020-07-18T19:17:25.213Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2336", + "object_repr": "Post-2336", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 12, + "fields": { + "action_time": "2020-07-18T19:17:40.596Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2336", + "object_repr": "Post-2336", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 13, + "fields": { + "action_time": "2020-07-19T10:55:55.807Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2764", + "object_repr": "Post-2764", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 14, + "fields": { + "action_time": "2020-07-19T10:57:40.643Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2764", + "object_repr": "Post-2764", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 15, + "fields": { + "action_time": "2020-07-19T10:58:05.823Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2764", + "object_repr": "Post-2764", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 16, + "fields": { + "action_time": "2020-07-26T09:51:52.478Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 17, + "fields": { + "action_time": "2020-07-26T09:52:04.691Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"password\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 18, + "fields": { + "action_time": "2020-07-26T09:52:12.392Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 19, + "fields": { + "action_time": "2020-07-26T09:56:15.949Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" + } +} +] diff --git a/src/newsreader/fixtures/local/fixture.json b/src/newsreader/fixtures/local/fixture.json index ffcc4fd..99176b5 100644 --- a/src/newsreader/fixtures/local/fixture.json +++ b/src/newsreader/fixtures/local/fixture.json @@ -47,7 +47,7 @@ "user" : 2, "succeeded" : true, "modified" : "2019-07-20T11:28:16.473Z", - "last_suceeded" : "2019-07-20T11:28:16.316Z", + "last_run" : "2019-07-20T11:28:16.316Z", "name" : "Hackers News", "website_url" : null, "created" : "2019-07-14T13:08:10.374Z", @@ -65,7 +65,7 @@ "error" : null, "user" : 2, "succeeded" : true, - "last_suceeded" : "2019-07-20T11:28:15.691Z", + "last_run" : "2019-07-20T11:28:15.691Z", "name" : "BBC", "modified" : "2019-07-20T12:07:49.164Z", "timezone" : "UTC", @@ -85,7 +85,7 @@ "website_url" : null, "name" : "Ars Technica", "succeeded" : true, - "last_suceeded" : "2019-07-20T11:28:15.986Z", + "last_run" : "2019-07-20T11:28:15.986Z", "modified" : "2019-07-20T11:28:16.033Z", "user" : 2 }, @@ -102,7 +102,7 @@ "user" : 2, "name" : "The Guardian", "succeeded" : true, - "last_suceeded" : "2019-07-20T11:28:16.078Z", + "last_run" : "2019-07-20T11:28:16.078Z", "modified" : "2019-07-20T12:07:44.292Z", "created" : "2019-07-20T11:25:02.089Z", "website_url" : null, @@ -119,7 +119,7 @@ "website_url" : null, "created" : "2019-07-20T11:25:30.121Z", "user" : 2, - "last_suceeded" : "2019-07-20T11:28:15.860Z", + "last_run" : "2019-07-20T11:28:15.860Z", "succeeded" : true, "modified" : "2019-07-20T12:07:28.473Z", "name" : "Tweakers" @@ -139,7 +139,7 @@ "website_url" : null, "timezone" : "UTC", "user" : 2, - "last_suceeded" : "2019-07-20T11:28:16.034Z", + "last_run" : "2019-07-20T11:28:16.034Z", "succeeded" : true, "modified" : "2019-07-20T12:07:21.704Z", "name" : "The Verge" diff --git a/src/newsreader/js/pages/categories/App.js b/src/newsreader/js/pages/categories/App.js index 691aaed..a035b46 100644 --- a/src/newsreader/js/pages/categories/App.js +++ b/src/newsreader/js/pages/categories/App.js @@ -69,6 +69,7 @@ class App extends React.Component { key={category.pk} category={category} showDialog={this.selectCategory} + updateUrl={this.props.updateUrl} /> ); }); @@ -80,7 +81,7 @@ class App extends React.Component { const pageHeader = ( <>

Categories

- + Create category diff --git a/src/newsreader/js/pages/categories/components/CategoryCard.js b/src/newsreader/js/pages/categories/components/CategoryCard.js index 94bd6f4..2e7cad4 100644 --- a/src/newsreader/js/pages/categories/components/CategoryCard.js +++ b/src/newsreader/js/pages/categories/components/CategoryCard.js @@ -33,7 +33,7 @@ const CategoryCard = props => { <> Edit diff --git a/src/newsreader/js/pages/categories/index.js b/src/newsreader/js/pages/categories/index.js index 9d75bb9..791fdbd 100644 --- a/src/newsreader/js/pages/categories/index.js +++ b/src/newsreader/js/pages/categories/index.js @@ -9,5 +9,15 @@ if (page) { const dataScript = document.getElementById('categories-data'); const categories = JSON.parse(dataScript.textContent); - ReactDOM.render(, page); + let createUrl = document.getElementById('createUrl').textContent; + let updateUrl = document.getElementById('updateUrl').textContent; + + ReactDOM.render( + , + page + ); } diff --git a/src/newsreader/js/pages/homepage/App.js b/src/newsreader/js/pages/homepage/App.js index 91cfa4e..77b6222 100644 --- a/src/newsreader/js/pages/homepage/App.js +++ b/src/newsreader/js/pages/homepage/App.js @@ -19,7 +19,11 @@ class App extends React.Component { return ( <> - + {this.props.error && ( @@ -30,6 +34,10 @@ class App extends React.Component { post={this.props.post} rule={this.props.rule} category={this.props.category} + feedUrl={this.props.feedUrl} + subredditUrl={this.props.subredditUrl} + timelineUrl={this.props.timelineUrl} + categoriesUrl={this.props.categoriesUrl} /> )} diff --git a/src/newsreader/js/pages/homepage/components/PostModal.js b/src/newsreader/js/pages/homepage/components/PostModal.js index 08033bc..5196102 100644 --- a/src/newsreader/js/pages/homepage/components/PostModal.js +++ b/src/newsreader/js/pages/homepage/components/PostModal.js @@ -3,7 +3,13 @@ import { connect } from 'react-redux'; import Cookies from 'js-cookie'; import { unSelectPost, markPostRead } from '../actions/posts.js'; -import { CATEGORY_TYPE, RULE_TYPE, FEED, SUBREDDIT } from '../constants.js'; +import { + CATEGORY_TYPE, + RULE_TYPE, + FEED, + SUBREDDIT, + TWITTER_TIMELINE, +} from '../constants.js'; import { formatDatetime } from '../../../utils.js'; class PostModal extends React.Component { @@ -44,10 +50,15 @@ class PostModal extends React.Component { const post = this.props.post; const publicationDate = formatDatetime(post.publicationDate); const titleClassName = post.read ? 'post__title post__title--read' : 'post__title'; - const ruleUrl = - this.props.rule.type === FEED - ? `/collection/rules/${this.props.rule.id}/` - : `/collection/rules/subreddits/${this.props.rule.id}/`; + let ruleUrl = ''; + + if (this.props.rule.type === SUBREDDIT) { + ruleUrl = `${this.props.subredditUrl}/${this.props.rule.id}/`; + } else if (this.props.rule.type === TWITTER_TIMELINE) { + ruleUrl = `${this.props.timelineUrl}/${this.props.rule.id}/`; + } else { + ruleUrl = `${this.props.feedUrl}/${this.props.rule.id}/`; + } return (
@@ -66,7 +77,7 @@ class PostModal extends React.Component { {this.props.category && ( diff --git a/src/newsreader/js/pages/homepage/components/postlist/PostItem.js b/src/newsreader/js/pages/homepage/components/postlist/PostItem.js index 9b64289..f69a463 100644 --- a/src/newsreader/js/pages/homepage/components/postlist/PostItem.js +++ b/src/newsreader/js/pages/homepage/components/postlist/PostItem.js @@ -1,7 +1,13 @@ import React from 'react'; import { connect } from 'react-redux'; -import { CATEGORY_TYPE, RULE_TYPE, FEED, SUBREDDIT } from '../../constants.js'; +import { + CATEGORY_TYPE, + RULE_TYPE, + FEED, + SUBREDDIT, + TWITTER_TIMELINE, +} from '../../constants.js'; import { selectPost } from '../../actions/posts.js'; import { formatDatetime } from '../../../../utils.js'; @@ -13,11 +19,15 @@ class PostItem extends React.Component { const titleClassName = post.read ? 'posts__header posts__header--read' : 'posts__header'; + let ruleUrl = ''; - const ruleUrl = - rule.type === FEED - ? `/collection/rules/${rule.id}/` - : `/collection/rules/subreddits/${rule.id}/`; + if (rule.type === SUBREDDIT) { + ruleUrl = `${this.props.subredditUrl}/${rule.id}/`; + } else if (rule.type === TWITTER_TIMELINE) { + ruleUrl = `${this.props.timelineUrl}/${rule.id}/`; + } else { + ruleUrl = `${this.props.feedUrl}/${rule.id}/`; + } return (
  • diff --git a/src/newsreader/js/pages/homepage/components/postlist/PostList.js b/src/newsreader/js/pages/homepage/components/postlist/PostList.js index cd57d6d..cff2437 100644 --- a/src/newsreader/js/pages/homepage/components/postlist/PostList.js +++ b/src/newsreader/js/pages/homepage/components/postlist/PostList.js @@ -38,7 +38,16 @@ class PostList extends React.Component { render() { const postItems = this.props.postsBySection.map((item, index) => { - return ; + return ( + + ); }); if (isEqual(this.props.selected, {})) { diff --git a/src/newsreader/js/pages/homepage/constants.js b/src/newsreader/js/pages/homepage/constants.js index 66b6365..22184b9 100644 --- a/src/newsreader/js/pages/homepage/constants.js +++ b/src/newsreader/js/pages/homepage/constants.js @@ -3,3 +3,4 @@ export const CATEGORY_TYPE = 'CATEGORY'; export const SUBREDDIT = 'subreddit'; export const FEED = 'feed'; +export const TWITTER_TIMELINE = 'twitter_timeline'; diff --git a/src/newsreader/js/pages/homepage/index.js b/src/newsreader/js/pages/homepage/index.js index c16ed39..394a06c 100644 --- a/src/newsreader/js/pages/homepage/index.js +++ b/src/newsreader/js/pages/homepage/index.js @@ -11,9 +11,19 @@ const page = document.getElementById('homepage--page'); if (page) { const store = configureStore(); + let feedUrl = document.getElementById('feedUrl').textContent; + let subredditUrl = document.getElementById('subredditUrl').textContent; + let timelineUrl = document.getElementById('timelineUrl').textContent; + let categoriesUrl = document.getElementById('categoriesUrl').textContent; + ReactDOM.render( - + , page ); diff --git a/src/newsreader/news/collection/admin.py b/src/newsreader/news/collection/admin.py index c5a7c5c..ece5c23 100644 --- a/src/newsreader/news/collection/admin.py +++ b/src/newsreader/news/collection/admin.py @@ -6,14 +6,7 @@ from newsreader.news.collection.models import CollectionRule class CollectionRuleAdmin(admin.ModelAdmin): fields = ("url", "name", "timezone", "category", "favicon", "user") - list_display = ( - "name", - "type_display", - "category", - "url", - "last_suceeded", - "succeeded", - ) + list_display = ("name", "type_display", "category", "url", "last_run", "succeeded") list_filter = ("user",) def save_model(self, request, obj, form, change): diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index f980191..7286526 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -1,7 +1,10 @@ -from bs4 import BeautifulSoup +import bleach -from newsreader.news.collection.exceptions import StreamParseException -from newsreader.news.collection.utils import fetch +from newsreader.news.collection.constants import ( + WHITELISTED_ATTRIBUTES, + WHITELISTED_TAGS, +) +from newsreader.news.core.models import Post class Stream: @@ -20,19 +23,16 @@ class Stream: def parse(self, response): raise NotImplementedError - class Meta: - abstract = True - class Client: """ - Retrieves the data with streams + Retrieves the data through streams """ stream = Stream def __init__(self, rules=[]): - self.rules = rules if rules else CollectionRule.objects.enabled() + self.rules = rules def __enter__(self): for rule in self.rules: @@ -43,36 +43,40 @@ class Client: def __exit__(self, *args, **kwargs): pass - class Meta: - abstract = True - class Builder: """ - Creates the collected posts + Builds instances of various types """ instances = [] stream = None + payload = None - def __init__(self, stream): + def __init__(self, payload, stream): + self.payload = payload self.stream = stream def __enter__(self): - self.create_posts(self.stream) return self def __exit__(self, *args, **kwargs): pass - def create_posts(self, stream): - pass + def build(self): + raise NotImplementedError - def save(self): - pass + def sanitize_fragment(self, fragment): + if not fragment: + return "" - class Meta: - abstract = True + return bleach.clean( + fragment, + tags=WHITELISTED_TAGS, + attributes=WHITELISTED_ATTRIBUTES, + strip=True, + strip_comments=True, + ) class Collector: @@ -88,46 +92,54 @@ class Collector: self.builder = builder if builder else self.builder def collect(self, rules=None): - with self.client(rules=rules) as client: - for data, stream in client: - with self.builder((data, stream)) as builder: - builder.save() - - class Meta: - abstract = True + raise NotImplementedError -class WebsiteStream(Stream): - def __init__(self, url): - self.url = url +class Scheduler: + """ + Schedules rules according to certain ratelimitting + """ - def read(self): - response = fetch(self.url) - - return (self.parse(response.content), self) - - def parse(self, payload): - try: - return BeautifulSoup(payload, "lxml") - except TypeError: - raise StreamParseException("Could not parse given HTML") + def get_scheduled_rules(self): + raise NotImplementedError -class URLBuilder(Builder): +class PostBuilder(Builder): + rule_type = None + def __enter__(self): - return self + self.existing_posts = { + post.remote_identifier: post + for post in Post.objects.filter( + rule=self.stream.rule, rule__type=self.rule_type + ) + } - def build(self): - data, stream = self.stream - rule = stream.rule + return super().__enter__() - try: - url = data["feed"]["link"] - except (KeyError, TypeError): - url = None + def save(self): + for post in self.instances: + post.save() - if url: - rule.website_url = url - rule.save() - return rule, url +class PostStream(Stream): + rule_type = None + + +class PostClient(Client): + stream = PostStream + + def set_rule_error(self, rule, exception): + length = rule._meta.get_field("error").max_length + + rule.error = exception.message[-length:] + rule.succeeded = False + + +class PostCollector(Collector): + def collect(self, rules=[]): + with self.client(rules=rules) as client: + for payload, stream in client: + with self.builder(payload, stream) as builder: + builder.build() + builder.save() diff --git a/src/newsreader/news/collection/choices.py b/src/newsreader/news/collection/choices.py index 65f7ef5..612079c 100644 --- a/src/newsreader/news/collection/choices.py +++ b/src/newsreader/news/collection/choices.py @@ -5,3 +5,10 @@ from django.utils.translation import gettext as _ class RuleTypeChoices(TextChoices): feed = "feed", _("Feed") subreddit = "subreddit", _("Subreddit") + twitter_timeline = "twitter_timeline", _("Twitter timeline") + + +class TwitterPostTypeChoices(TextChoices): + photo = "photo", _("Photo") + video = "video", _("Video") + animated_gif = "animated_gif", _("GIF") diff --git a/src/newsreader/news/collection/constants.py b/src/newsreader/news/collection/constants.py index eade898..0c73642 100644 --- a/src/newsreader/news/collection/constants.py +++ b/src/newsreader/news/collection/constants.py @@ -23,6 +23,7 @@ WHITELISTED_TAGS = ( WHITELISTED_ATTRIBUTES = { **BLEACH_ATTRIBUTES, "a": ["href", "rel"], - "img": ["alt", "src"], - "source": ["srcset", "media", "src", "type"], + "img": ["alt", "src", "loading"], + "video": ["controls", "muted"], + "source": ["srcset", "src", "media", "type"], } diff --git a/src/newsreader/news/collection/favicon.py b/src/newsreader/news/collection/favicon.py index 44b96bf..639e7f6 100644 --- a/src/newsreader/news/collection/favicon.py +++ b/src/newsreader/news/collection/favicon.py @@ -1,16 +1,12 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import urljoin, urlparse -from newsreader.news.collection.base import ( - Builder, - Client, - Collector, - Stream, - URLBuilder, - WebsiteStream, -) -from newsreader.news.collection.exceptions import StreamException +from bs4 import BeautifulSoup + +from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.exceptions import StreamException, StreamParseException from newsreader.news.collection.feed import FeedClient +from newsreader.news.collection.utils import fetch LINK_RELS = [ @@ -21,17 +17,45 @@ LINK_RELS = [ ] +class WebsiteStream(Stream): + def read(self): + response = fetch(self.rule.website_url) + + return self.parse(response.content), self + + def parse(self, payload): + try: + return BeautifulSoup(payload, features="lxml") + except TypeError: + raise StreamParseException("Could not parse given HTML") + + +class WebsiteURLBuilder(Builder): + def build(self): + try: + url = self.payload["feed"]["link"] + except (KeyError, TypeError): + url = None + + self.instances = [(self.stream, url)] if url else [] + + def save(self): + for stream, url in self.instances: + stream.rule.website_url = url + stream.rule.save() + + class FaviconBuilder(Builder): def build(self): - rule, soup = self.stream + rule = self.stream.rule - url = self.parse(soup, rule.website_url) + url = self.parse() - if url: - rule.favicon = url - rule.save() + self.instances = [(rule, url)] if url else [] + + def parse(self): + soup = self.payload - def parse(self, soup, website_url): if not soup.head: return @@ -44,9 +68,9 @@ class FaviconBuilder(Builder): parsed_url = urlparse(url) if not parsed_url.scheme and not parsed_url.netloc: - if not website_url: + if not self.stream.rule.website_url: return - return urljoin(website_url, url) + return urljoin(self.stream.rule.website_url, url) elif not parsed_url.scheme: return urljoin(f"https://{parsed_url.netloc}", parsed_url.path) @@ -73,6 +97,11 @@ class FaviconBuilder(Builder): elif icons: return icons.pop() + def save(self): + for rule, favicon_url in self.instances: + rule.favicon = favicon_url + rule.save() + class FaviconClient(Client): stream = WebsiteStream @@ -82,39 +111,35 @@ class FaviconClient(Client): def __enter__(self): with ThreadPoolExecutor(max_workers=10) as executor: - futures = { - executor.submit(stream.read): rule for rule, stream in self.streams - } + futures = [executor.submit(stream.read) for stream in self.streams] for future in as_completed(futures): - rule = futures[future] - try: - response_data, stream = future.result() + payload, stream = future.result() except StreamException: continue - yield (rule, response_data) + yield payload, stream class FaviconCollector(Collector): feed_client, favicon_client = (FeedClient, FaviconClient) - url_builder, favicon_builder = (URLBuilder, FaviconBuilder) + url_builder, favicon_builder = (WebsiteURLBuilder, FaviconBuilder) def collect(self, rules=None): streams = [] with self.feed_client(rules=rules) as client: - for data, stream in client: - with self.url_builder((data, stream)) as builder: - rule, url = builder.build() + for payload, stream in client: + with self.url_builder(payload, stream) as builder: + builder.build() + builder.save() - if not url: - continue - - streams.append((rule, WebsiteStream(url))) + if builder.instances: + streams.append(WebsiteStream(stream.rule)) with self.favicon_client(streams) as client: - for rule, data in client: - with self.favicon_builder((rule, data)) as builder: + for payload, stream in client: + with self.favicon_builder(payload, stream) as builder: builder.build() + builder.save() diff --git a/src/newsreader/news/collection/feed.py b/src/newsreader/news/collection/feed.py index f67a109..ae6cd42 100644 --- a/src/newsreader/news/collection/feed.py +++ b/src/newsreader/news/collection/feed.py @@ -6,17 +6,17 @@ from datetime import timedelta from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.utils import timezone -import bleach import pytz from feedparser import parse -from newsreader.news.collection.base import Builder, Client, Collector, Stream -from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.constants import ( - WHITELISTED_ATTRIBUTES, - WHITELISTED_TAGS, +from newsreader.news.collection.base import ( + PostBuilder, + PostClient, + PostCollector, + PostStream, ) +from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.exceptions import ( StreamDeniedException, StreamException, @@ -24,7 +24,6 @@ from newsreader.news.collection.exceptions import ( StreamParseException, StreamTimeOutException, ) -from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.utils import ( build_publication_date, fetch, @@ -36,32 +35,10 @@ from newsreader.news.core.models import Post logger = logging.getLogger(__name__) -class FeedBuilder(Builder): - instances = [] +class FeedBuilder(PostBuilder): + rule__type = RuleTypeChoices.feed - def __enter__(self): - _, stream = self.stream - - self.instances = [] - self.existing_posts = { - post.remote_identifier: post - for post in Post.objects.filter( - rule=stream.rule, rule__type=RuleTypeChoices.feed - ) - } - - return super().__enter__() - - def create_posts(self, stream): - data, stream = stream - - with FeedDuplicateHandler(stream.rule) as duplicate_handler: - entries = data.get("entries", []) - - instances = self.build(entries, stream.rule) - self.instances = duplicate_handler.check(instances) - - def build(self, entries, rule): + def build(self): field_mapping = { "id": "remote_identifier", "title": "title", @@ -70,56 +47,47 @@ class FeedBuilder(Builder): "published_parsed": "publication_date", "author": "author", } + tz = pytz.timezone(self.stream.rule.timezone) + instances = [] - tz = pytz.timezone(rule.timezone) + with FeedDuplicateHandler(self.stream.rule) as duplicate_handler: + entries = self.payload.get("entries", []) - for entry in entries: - data = {"rule_id": rule.pk} + for entry in entries: + data = {"rule_id": self.stream.rule.pk} - for field, model_field in field_mapping.items(): - if not field in entry: - continue + for field, model_field in field_mapping.items(): + if not field in entry: + continue - value = truncate_text(Post, model_field, entry[field]) + value = truncate_text(Post, model_field, entry[field]) - if field == "published_parsed": - data[model_field] = build_publication_date(value, tz) - elif field == "summary": - data[model_field] = self.sanitize_fragment(value) - else: - data[model_field] = value + if field == "published_parsed": + data[model_field] = build_publication_date(value, tz) + elif field == "summary": + data[model_field] = self.sanitize_fragment(value) + else: + data[model_field] = value - if "content" in entry: - content = self.get_content(entry["content"]) - body = data.get("body", "") + if "content" in entry: + content = self.get_content(entry["content"]) + body = data.get("body", "") - if not body or len(body) < len(content): - data["body"] = content + if not body or len(body) < len(content): + data["body"] = content - yield Post(**data) + instances.append(Post(**data)) - def sanitize_fragment(self, fragment): - if not fragment: - return "" - - return bleach.clean( - fragment, - tags=WHITELISTED_TAGS, - attributes=WHITELISTED_ATTRIBUTES, - strip=True, - strip_comments=True, - ) + self.instances = duplicate_handler.check(instances) def get_content(self, items): content = "\n ".join([item.get("value") for item in items]) return self.sanitize_fragment(content) - def save(self): - for post in self.instances: - post.save() +class FeedStream(PostStream): + rule_type = RuleTypeChoices.feed -class FeedStream(Stream): def read(self): response = fetch(self.rule.url) @@ -133,17 +101,9 @@ class FeedStream(Stream): raise StreamParseException(response=response, message=message) from e -class FeedClient(Client): +class FeedClient(PostClient): stream = FeedStream - def __init__(self, rules=[]): - if rules: - self.rules = rules - else: - self.rules = CollectionRule.objects.filter( - enabled=True, type=RuleTypeChoices.feed - ) - def __enter__(self): streams = [self.stream(rule) for rule in self.rules] @@ -154,13 +114,12 @@ class FeedClient(Client): stream = futures[future] try: - response_data = future.result() + payload = future.result() stream.rule.error = None stream.rule.succeeded = True - stream.rule.last_suceeded = timezone.now() - yield response_data + yield payload except (StreamNotFoundException, StreamTimeOutException) as e: logger.warning(f"Request failed for {stream.rule.url}") @@ -174,16 +133,11 @@ class FeedClient(Client): continue finally: + stream.rule.last_run = timezone.now() stream.rule.save() - def set_rule_error(self, rule, exception): - length = rule._meta.get_field("error").max_length - rule.error = exception.message[-length:] - rule.succeeded = False - - -class FeedCollector(Collector): +class FeedCollector(PostCollector): builder = FeedBuilder client = FeedClient diff --git a/src/newsreader/news/collection/forms.py b/src/newsreader/news/collection/forms.py deleted file mode 100644 index c79a867..0000000 --- a/src/newsreader/news/collection/forms.py +++ /dev/null @@ -1,101 +0,0 @@ -from django import forms -from django.core.exceptions import ValidationError -from django.utils.safestring import mark_safe -from django.utils.translation import gettext_lazy as _ - -import pytz - -from newsreader.core.forms import CheckboxInput -from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.models import CollectionRule -from newsreader.news.collection.reddit import REDDIT_API_URL -from newsreader.news.core.models import Category - - -def get_reddit_help_text(): - return mark_safe( - "Only subreddits are supported" - " see the 'listings' section in the reddit API docs." - " For example: https://oauth.reddit.com/r/aww" - ) - - -class CollectionRuleForm(forms.ModelForm): - category = forms.ModelChoiceField(required=False, queryset=Category.objects.all()) - timezone = forms.ChoiceField( - widget=forms.Select(attrs={"size": len(pytz.all_timezones)}), - choices=((timezone, timezone) for timezone in pytz.all_timezones), - help_text=_("The timezone which the feed uses"), - initial=pytz.utc, - ) - - def __init__(self, *args, **kwargs): - self.user = kwargs.pop("user") - - super().__init__(*args, **kwargs) - - self.fields["category"].queryset = Category.objects.filter(user=self.user) - - def save(self, commit=True): - instance = super().save(commit=False) - instance.user = self.user - - if commit: - instance.save() - self.save_m2m() - - return instance - - class Meta: - model = CollectionRule - fields = ("name", "url", "timezone", "favicon", "category") - - -class CollectionRuleBulkForm(forms.Form): - rules = forms.ModelMultipleChoiceField(queryset=CollectionRule.objects.none()) - - def __init__(self, user, *args, **kwargs): - self.user = user - - super().__init__(*args, **kwargs) - - self.fields["rules"].queryset = CollectionRule.objects.filter(user=user) - - -class SubRedditRuleForm(CollectionRuleForm): - url = forms.URLField(max_length=1024, help_text=get_reddit_help_text) - - timezone = None - - def clean_url(self): - url = self.cleaned_data["url"] - - if not url.startswith(REDDIT_API_URL): - raise ValidationError(_("This does not look like an Reddit API URL")) - - return url - - def save(self, commit=True): - instance = super().save(commit=False) - - instance.type = RuleTypeChoices.subreddit - instance.timezone = str(pytz.utc) - - if commit: - instance.save() - self.save_m2m() - - return instance - - class Meta: - model = CollectionRule - fields = ("name", "url", "favicon", "category") - - -class OPMLImportForm(forms.Form): - file = forms.FileField(allow_empty_file=False) - skip_existing = forms.BooleanField( - initial=False, required=False, widget=CheckboxInput - ) diff --git a/src/newsreader/news/collection/forms/__init__.py b/src/newsreader/news/collection/forms/__init__.py new file mode 100644 index 0000000..88a51c7 --- /dev/null +++ b/src/newsreader/news/collection/forms/__init__.py @@ -0,0 +1,4 @@ +from newsreader.news.collection.forms.feed import FeedForm, OPMLImportForm +from newsreader.news.collection.forms.reddit import SubRedditForm +from newsreader.news.collection.forms.rules import CollectionRuleBulkForm +from newsreader.news.collection.forms.twitter import TwitterTimelineForm diff --git a/src/newsreader/news/collection/forms/base.py b/src/newsreader/news/collection/forms/base.py new file mode 100644 index 0000000..da23659 --- /dev/null +++ b/src/newsreader/news/collection/forms/base.py @@ -0,0 +1,29 @@ +from django import forms + +from newsreader.news.collection.models import CollectionRule +from newsreader.news.core.models import Category + + +class CollectionRuleForm(forms.ModelForm): + category = forms.ModelChoiceField(required=False, queryset=Category.objects.all()) + + def __init__(self, *args, **kwargs): + self.user = kwargs.pop("user") + + super().__init__(*args, **kwargs) + + self.fields["category"].queryset = Category.objects.filter(user=self.user) + + def save(self, commit=True): + instance = super().save(commit=False) + instance.user = self.user + + if commit: + instance.save() + self.save_m2m() + + return instance + + class Meta: + model = CollectionRule + fields = "__all__" diff --git a/src/newsreader/news/collection/forms/feed.py b/src/newsreader/news/collection/forms/feed.py new file mode 100644 index 0000000..4a22a2e --- /dev/null +++ b/src/newsreader/news/collection/forms/feed.py @@ -0,0 +1,28 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +import pytz + +from newsreader.core.forms import CheckboxInput +from newsreader.news.collection.forms.base import CollectionRuleForm +from newsreader.news.collection.models import CollectionRule + + +class FeedForm(CollectionRuleForm): + timezone = forms.ChoiceField( + widget=forms.Select(attrs={"size": len(pytz.all_timezones)}), + choices=((timezone, timezone) for timezone in pytz.all_timezones), + help_text=_("The timezone which the feed uses"), + initial=pytz.utc, + ) + + class Meta: + model = CollectionRule + fields = ("name", "url", "timezone", "favicon", "category") + + +class OPMLImportForm(forms.Form): + file = forms.FileField(allow_empty_file=False) + skip_existing = forms.BooleanField( + initial=False, required=False, widget=CheckboxInput + ) diff --git a/src/newsreader/news/collection/forms/reddit.py b/src/newsreader/news/collection/forms/reddit.py new file mode 100644 index 0000000..1744893 --- /dev/null +++ b/src/newsreader/news/collection/forms/reddit.py @@ -0,0 +1,49 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +import pytz + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms.base import CollectionRuleForm +from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.reddit import REDDIT_API_URL + + +def get_reddit_help_text(): + return mark_safe( + "Only subreddits are supported" + " see the 'listings' section in the reddit API docs." + " For example: https://oauth.reddit.com/r/aww" + ) + + +class SubRedditForm(CollectionRuleForm): + url = forms.URLField(max_length=1024, help_text=get_reddit_help_text) + + def clean_url(self): + url = self.cleaned_data["url"] + + if not url.startswith(REDDIT_API_URL): + raise ValidationError(_("This does not look like an Reddit API URL")) + + return url + + def save(self, commit=True): + instance = super().save(commit=False) + + instance.type = RuleTypeChoices.subreddit + instance.timezone = str(pytz.utc) + + if commit: + instance.save() + self.save_m2m() + + return instance + + class Meta: + model = CollectionRule + fields = ("name", "url", "favicon", "category") diff --git a/src/newsreader/news/collection/forms/rules.py b/src/newsreader/news/collection/forms/rules.py new file mode 100644 index 0000000..fade945 --- /dev/null +++ b/src/newsreader/news/collection/forms/rules.py @@ -0,0 +1,14 @@ +from django import forms + +from newsreader.news.collection.models import CollectionRule + + +class CollectionRuleBulkForm(forms.Form): + rules = forms.ModelMultipleChoiceField(queryset=CollectionRule.objects.none()) + + def __init__(self, user, *args, **kwargs): + self.user = user + + super().__init__(*args, **kwargs) + + self.fields["rules"].queryset = CollectionRule.objects.filter(user=user) diff --git a/src/newsreader/news/collection/forms/twitter.py b/src/newsreader/news/collection/forms/twitter.py new file mode 100644 index 0000000..902652b --- /dev/null +++ b/src/newsreader/news/collection/forms/twitter.py @@ -0,0 +1,35 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +import pytz + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms.base import CollectionRuleForm +from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.twitter import TWITTER_API_URL + + +class TwitterTimelineForm(CollectionRuleForm): + screen_name = forms.CharField( + max_length=255, + label=_("Twitter profile name"), + help_text=_("Profile name without hashtags"), + required=True, + ) + + def save(self, commit=True): + instance = super().save(commit=False) + + instance.type = RuleTypeChoices.twitter_timeline + instance.timezone = str(pytz.utc) + instance.url = f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name={instance.screen_name}&tweet_mode=extended" + + if commit: + instance.save() + self.save_m2m() + + return instance + + class Meta: + model = CollectionRule + fields = ("name", "screen_name", "favicon", "category") diff --git a/src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py b/src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py new file mode 100644 index 0000000..2ce4cb3 --- /dev/null +++ b/src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.7 on 2020-08-07 18:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("collection", "0008_collectionrule_type")] + + operations = [ + migrations.AddField( + model_name="collectionrule", + name="screen_name", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name="collectionrule", + name="type", + field=models.CharField( + choices=[ + ("feed", "Feed"), + ("subreddit", "Subreddit"), + ("twitter", "Twitter"), + ], + default="feed", + max_length=20, + ), + ), + ] diff --git a/src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py b/src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py new file mode 100644 index 0000000..2f08f6e --- /dev/null +++ b/src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-09-13 19:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("collection", "0009_auto_20200807_2030")] + + operations = [ + migrations.AlterField( + model_name="collectionrule", + name="type", + field=models.CharField( + choices=[ + ("feed", "Feed"), + ("subreddit", "Subreddit"), + ("twitter_timeline", "Twitter timeline"), + ], + default="feed", + max_length=20, + ), + ) + ] diff --git a/src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py b/src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py new file mode 100644 index 0000000..308c654 --- /dev/null +++ b/src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-09-13 19:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("collection", "0010_auto_20200913_2101")] + + operations = [ + migrations.RenameField( + model_name="collectionrule", old_name="last_suceeded", new_name="last_run" + ) + ] diff --git a/src/newsreader/news/collection/models.py b/src/newsreader/news/collection/models.py index 35841ba..92dfe51 100644 --- a/src/newsreader/news/collection/models.py +++ b/src/newsreader/news/collection/models.py @@ -41,9 +41,8 @@ class CollectionRule(TimeStampedModel): on_delete=models.SET_NULL, ) - last_suceeded = models.DateTimeField(blank=True, null=True) + last_run = models.DateTimeField(blank=True, null=True) succeeded = models.BooleanField(default=False) - error = models.CharField(max_length=1024, blank=True, null=True) enabled = models.BooleanField( @@ -57,6 +56,9 @@ class CollectionRule(TimeStampedModel): on_delete=models.CASCADE, ) + # Twitter + screen_name = models.CharField(max_length=255, blank=True, null=True) + objects = CollectionRuleQuerySet.as_manager() def __str__(self): @@ -66,5 +68,13 @@ class CollectionRule(TimeStampedModel): def update_url(self): if self.type == RuleTypeChoices.subreddit: return reverse("news:collection:subreddit-update", kwargs={"pk": self.pk}) + elif self.type == RuleTypeChoices.twitter_timeline: + return reverse( + "news:collection:twitter-timeline-update", kwargs={"pk": self.pk} + ) - return reverse("news:collection:rule-update", kwargs={"pk": self.pk}) + return reverse("news:collection:feed-update", kwargs={"pk": self.pk}) + + @property + def failed(self): + return not self.succeeded and self.last_run diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 557271c..daeb85f 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -12,11 +12,16 @@ from django.core.cache import cache from django.utils import timezone from django.utils.html import format_html -import bleach import pytz import requests -from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.base import ( + PostBuilder, + PostClient, + PostCollector, + PostStream, + Scheduler, +) from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.constants import ( WHITELISTED_ATTRIBUTES, @@ -93,32 +98,32 @@ def get_reddit_access_token(code, user): return response_data["access_token"], response_data["refresh_token"] -class RedditBuilder(Builder): - def __enter__(self): - _, stream = self.stream +# Note that the API always returns 204's with correct basic auth headers +def revoke_reddit_token(user): + client_auth = requests.auth.HTTPBasicAuth( + settings.REDDIT_CLIENT_ID, settings.REDDIT_CLIENT_SECRET + ) - self.instances = [] - self.existing_posts = { - post.remote_identifier: post - for post in Post.objects.filter( - rule=stream.rule, rule__type=RuleTypeChoices.subreddit - ) - } + response = post( + f"{REDDIT_URL}/api/v1/revoke_token", + data={"token": user.reddit_refresh_token, "token_type_hint": "refresh_token"}, + auth=client_auth, + ) - return super().__enter__() + return response.status_code == 204 - def create_posts(self, stream): - data, stream = stream - posts = [] - if not "data" in data or not "children" in data["data"]: +class RedditBuilder(PostBuilder): + rule_type = RuleTypeChoices.subreddit + + def build(self): + results = {} + + if not "data" in self.payload or not "children" in self.payload["data"]: return - posts = data["data"]["children"] - self.instances = self.build(posts, stream.rule) - - def build(self, posts, rule): - results = {} + posts = self.payload["data"]["children"] + rule = self.stream.rule for post in posts: if not "data" in post or post["kind"] != REDDIT_POST: @@ -139,17 +144,7 @@ class RedditBuilder(Builder): if is_text_post: uncleaned_body = data["selftext_html"] unescaped_body = unescape(uncleaned_body) if uncleaned_body else "" - body = ( - bleach.clean( - unescaped_body, - tags=WHITELISTED_TAGS, - attributes=WHITELISTED_ATTRIBUTES, - strip=True, - strip_comments=True, - ) - if unescaped_body - else "" - ) + body = self.sanitize_fragment(unescaped_body) if unescaped_body else "" elif direct_url.endswith(REDDIT_IMAGE_EXTENSIONS): body = format_html( "
    {title}
    ", @@ -192,7 +187,9 @@ class RedditBuilder(Builder): parsed_date = datetime.fromtimestamp(post["data"]["created_utc"]) created_date = pytz.utc.localize(parsed_date) except (OverflowError, OSError): - logging.warning(f"Failed parsing timestamp from {url_fragment}") + logging.warning( + f"Failed parsing timestamp from {REDDIT_URL}{post_url_fragment}" + ) created_date = timezone.now() post_data = { @@ -216,14 +213,98 @@ class RedditBuilder(Builder): results[remote_identifier] = Post(**post_data) - return results.values() - - def save(self): - for post in self.instances: - post.save() + self.instances = results.values() -class RedditScheduler: +class RedditStream(PostStream): + rule_type = RuleTypeChoices.subreddit + headers = {} + + def __init__(self, rule): + super().__init__(rule) + + self.headers = { + f"Authorization": f"bearer {self.rule.user.reddit_access_token}" + } + + def read(self): + response = fetch(self.rule.url, headers=self.headers) + + return self.parse(response), self + + def parse(self, response): + try: + return response.json() + except JSONDecodeError as e: + raise StreamParseException( + response=response, message="Failed parsing json" + ) from e + + +class RedditClient(PostClient): + stream = RedditStream + + def __enter__(self): + streams = [[self.stream(rule) for rule in batch] for batch in self.rules] + rate_limitted = False + + with ThreadPoolExecutor(max_workers=10) as executor: + for batch in streams: + futures = {executor.submit(stream.read): stream for stream in batch} + + if rate_limitted: + logger.warning("Aborting requests, ratelimit hit") + break + + for future in as_completed(futures): + stream = futures[future] + + try: + response_data = future.result() + + stream.rule.error = None + stream.rule.succeeded = True + + yield response_data + except StreamDeniedException as e: + logger.warning( + f"Access token expired for user {stream.rule.user.pk}" + ) + + stream.rule.user.reddit_access_token = None + stream.rule.user.save() + + self.set_rule_error(stream.rule, e) + + RedditTokenTask.delay(stream.rule.user.pk) + + break + except StreamTooManyException as e: + logger.exception("Ratelimit hit, aborting batched subreddits") + + self.set_rule_error(stream.rule, e) + + rate_limitted = True + break + except StreamException as e: + logger.exception( + f"Stream failed reading content from {stream.rule.url}" + ) + + self.set_rule_error(stream.rule, e) + + continue + finally: + stream.rule.last_run = timezone.now() + stream.rule.save() + + +class RedditCollector(PostCollector): + builder = RedditBuilder + client = RedditClient + + +class RedditScheduler(Scheduler): max_amount = RATE_LIMIT max_user_amount = RATE_LIMIT / 4 @@ -234,7 +315,7 @@ class RedditScheduler: user__reddit_access_token__isnull=False, user__reddit_refresh_token__isnull=False, enabled=True, - ).order_by("last_suceeded")[:200] + ).order_by("last_run")[:200] else: self.subreddits = subreddits @@ -263,100 +344,3 @@ class RedditScheduler: current_amount += 1 return list(rule_mapping.values()) - - -class RedditStream(Stream): - headers = {} - user = None - - def __init__(self, rule): - super().__init__(rule) - - self.user = self.rule.user - self.headers = { - f"Authorization": f"bearer {self.rule.user.reddit_access_token}" - } - - def read(self): - response = fetch(self.rule.url, headers=self.headers) - - return self.parse(response), self - - def parse(self, response): - try: - return response.json() - except JSONDecodeError as e: - raise StreamParseException( - response=response, message=f"Failed parsing json" - ) from e - - -class RedditClient(Client): - stream = RedditStream - - def __init__(self, rules=[]): - self.rules = rules - - def __enter__(self): - streams = [[self.stream(rule) for rule in batch] for batch in self.rules] - rate_limitted = False - - with ThreadPoolExecutor(max_workers=10) as executor: - for batch in streams: - futures = {executor.submit(stream.read): stream for stream in batch} - - if rate_limitted: - break - - for future in as_completed(futures): - stream = futures[future] - - try: - response_data = future.result() - - stream.rule.error = None - stream.rule.succeeded = True - stream.rule.last_suceeded = timezone.now() - - yield response_data - except StreamDeniedException as e: - logger.warning( - f"Access token expired for user {stream.user.pk}" - ) - - stream.rule.user.reddit_access_token = None - stream.rule.user.save() - - self.set_rule_error(stream.rule, e) - - RedditTokenTask.delay(stream.rule.user.pk) - - break - except StreamTooManyException as e: - logger.exception("Ratelimit hit, aborting batched subreddits") - - self.set_rule_error(stream.rule, e) - - rate_limitted = True - break - except StreamException as e: - logger.exception( - "Stream failed reading content from " f"{stream.rule.url}" - ) - - self.set_rule_error(stream.rule, e) - - continue - finally: - stream.rule.save() - - def set_rule_error(self, rule, exception): - length = rule._meta.get_field("error").max_length - - rule.error = exception.message[-length:] - rule.succeeded = False - - -class RedditCollector(Collector): - builder = RedditBuilder - client = RedditClient diff --git a/src/newsreader/news/collection/tasks.py b/src/newsreader/news/collection/tasks.py index a04c5f9..926b05b 100644 --- a/src/newsreader/news/collection/tasks.py +++ b/src/newsreader/news/collection/tasks.py @@ -114,6 +114,40 @@ class RedditTokenTask(app.Task): user.save() +class TwitterTimelineTask(app.Task): + name = "TwitterTimelineTask" + ignore_result = True + + def run(self, user_pk): + from newsreader.news.collection.twitter import ( + TwitterCollector, + TwitterTimeLineScheduler, + ) + + try: + user = User.objects.get(pk=user_pk) + except ObjectDoesNotExist: + message = f"User {user_pk} does not exist" + logger.exception(message) + + raise Reject(reason=message, requeue=False) + + with MemCacheLock("f{user.email}-timeline-task", self.app.oid) as acquired: + if acquired: + logger.info(f"Running twitter timeline task for user {user_pk}") + + scheduler = TwitterTimeLineScheduler(user) + timelines = scheduler.get_scheduled_rules() + + collector = TwitterCollector() + collector.collect(rules=timelines) + else: + logger.warning(f"Cancelling task due to existing lock") + + raise Reject(reason="Task already running", requeue=False) + + FeedTask = app.register_task(FeedTask()) RedditTask = app.register_task(RedditTask()) RedditTokenTask = app.register_task(RedditTokenTask()) +TwitterTimelineTask = app.register_task(TwitterTimelineTask()) diff --git a/src/newsreader/news/collection/templates/news/collection/views/rule-create.html b/src/newsreader/news/collection/templates/news/collection/views/feed-create.html similarity index 78% rename from src/newsreader/news/collection/templates/news/collection/views/rule-create.html rename to src/newsreader/news/collection/templates/news/collection/views/feed-create.html index 82ed6c5..c24791a 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rule-create.html +++ b/src/newsreader/news/collection/templates/news/collection/views/feed-create.html @@ -4,6 +4,6 @@ {% block content %}
    {% url "news:collection:rules" as cancel_url %} - {% include "components/form/form.html" with form=form title="Create rule" cancel_url=cancel_url confirm_text="Create rule" %} + {% include "components/form/form.html" with form=form title="Add a feed" cancel_url=cancel_url confirm_text="Add feed" %}
    {% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/rule-update.html b/src/newsreader/news/collection/templates/news/collection/views/feed-update.html similarity index 72% rename from src/newsreader/news/collection/templates/news/collection/views/rule-update.html rename to src/newsreader/news/collection/templates/news/collection/views/feed-update.html index 0a705b8..33b1faf 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rule-update.html +++ b/src/newsreader/news/collection/templates/news/collection/views/feed-update.html @@ -3,12 +3,12 @@ {% block content %}
    - {% if rule.error %} + {% if feed.error %} {% trans "Failed to retrieve posts" as title %} - {% include "components/textbox/textbox.html" with title=title body=rule.error class="text-section--error" only %} + {% include "components/textbox/textbox.html" with title=title body=feed.error class="text-section--error" only %} {% endif %} {% url "news:collection:rules" as cancel_url %} - {% include "components/form/form.html" with form=form title="Update rule" cancel_url=cancel_url confirm_text="Save rule" only %} + {% include "components/form/form.html" with form=form title="Update feed" cancel_url=cancel_url confirm_text="Save feed" only %}
    {% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/import.html b/src/newsreader/news/collection/templates/news/collection/views/import.html index df19887..9719847 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/import.html +++ b/src/newsreader/news/collection/templates/news/collection/views/import.html @@ -4,6 +4,6 @@ {% block content %}
    {% url "news:collection:rules" as cancel_url %} - {% include "components/form/form.html" with form=form title="Import an OPML file" cancel_url=cancel_url confirm_text="Import rules" %} + {% include "components/form/form.html" with form=form title="Import an OPML file" cancel_url=cancel_url confirm_text="Import feeds" %}
    {% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/rules.html b/src/newsreader/news/collection/templates/news/collection/views/rules.html index 0cd1870..678716e 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rules.html +++ b/src/newsreader/news/collection/templates/news/collection/views/rules.html @@ -14,8 +14,9 @@ @@ -36,7 +37,7 @@ {% for rule in rules %} - + {% with rule|id_for_label:"rules" as id_for_label %} {% include "components/form/checkbox.html" with name="rules" value=rule.pk id=id_for_label id_for_label=id_for_label %} @@ -54,10 +55,10 @@ {{ rule.url }} - {% if rule.succeeded %} - - {% else %} + {% if rule.failed %} + {% else %} + {% endif %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html new file mode 100644 index 0000000..7c8eb13 --- /dev/null +++ b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% load static %} + +{% block content %} +
    + {% url "news:collection:rules" as cancel_url %} + {% include "components/form/form.html" with form=form title="Add a Twitter profile" cancel_url=cancel_url confirm_text="Add profile" %} +
    +{% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html new file mode 100644 index 0000000..51de47a --- /dev/null +++ b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{% load static i18n %} + +{% block content %} +
    + {% if timeline.error %} + {% trans "Failed to retrieve posts" as title %} + {% include "components/textbox/textbox.html" with title=title body=timeline.error class="text-section--error" only %} + {% endif %} + + {% url "news:collection:rules" as cancel_url %} + {% include "components/form/form.html" with form=form title="Update profile" cancel_url=cancel_url confirm_text="Save profile" %} +
    +{% endblock %} diff --git a/src/newsreader/news/collection/tests/factories.py b/src/newsreader/news/collection/tests/factories.py index fdf786f..26f66cc 100644 --- a/src/newsreader/news/collection/tests/factories.py +++ b/src/newsreader/news/collection/tests/factories.py @@ -28,3 +28,8 @@ class FeedFactory(CollectionRuleFactory): class SubredditFactory(CollectionRuleFactory): type = RuleTypeChoices.subreddit website_url = REDDIT_URL + + +class TwitterTimelineFactory(CollectionRuleFactory): + type = RuleTypeChoices.twitter_timeline + screen_name = factory.Faker("user_name") diff --git a/src/newsreader/news/collection/tests/favicon/builder/tests.py b/src/newsreader/news/collection/tests/favicon/builder/tests.py index e8a1a34..d21f77e 100644 --- a/src/newsreader/news/collection/tests/favicon/builder/tests.py +++ b/src/newsreader/news/collection/tests/favicon/builder/tests.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock + from django.test import TestCase from newsreader.news.collection.favicon import FaviconBuilder @@ -12,8 +14,11 @@ class FaviconBuilderTestCase(TestCase): def test_simple(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, simple_mock)) as builder: + with FaviconBuilder(simple_mock, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.bbc.com/favicon.ico") @@ -22,24 +27,33 @@ class FaviconBuilderTestCase(TestCase): website_url="https://www.theguardian.com/", favicon=None ) - with FaviconBuilder((rule, mock_without_url)) as builder: + with FaviconBuilder(mock_without_url, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.theguardian.com/favicon.ico") def test_without_header(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_without_header)) as builder: + with FaviconBuilder(mock_without_header, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, None) def test_weird_path(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_with_weird_path)) as builder: + with FaviconBuilder(mock_with_weird_path, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals( rule.favicon, "https://www.theguardian.com/jabadaba/doe/favicon.ico" @@ -48,15 +62,21 @@ class FaviconBuilderTestCase(TestCase): def test_other_url(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_with_other_url)) as builder: + with FaviconBuilder(mock_with_other_url, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.theguardian.com/icon.png") def test_url_with_favicon_takes_precedence(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_with_multiple_icons)) as builder: + with FaviconBuilder(mock_with_multiple_icons, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.bbc.com/favicon.ico") diff --git a/src/newsreader/news/collection/tests/favicon/client/tests.py b/src/newsreader/news/collection/tests/favicon/client/tests.py index 717ee0c..85b8fa3 100644 --- a/src/newsreader/news/collection/tests/favicon/client/tests.py +++ b/src/newsreader/news/collection/tests/favicon/client/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase @@ -19,22 +19,22 @@ class FaviconClientTestCase(TestCase): def test_simple(self): rule = CollectionRuleFactory() - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.return_value = (simple_mock, stream) - with FaviconClient([(rule, stream)]) as client: - for rule, data in client: - self.assertEquals(rule.pk, rule.pk) - self.assertEquals(data, simple_mock) + with FaviconClient([stream]) as client: + for payload, stream in client: + self.assertEquals(stream.rule.pk, rule.pk) + self.assertEquals(payload, simple_mock) stream.read.assert_called_once_with() def test_client_catches_stream_exception(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass @@ -46,10 +46,10 @@ class FaviconClientTestCase(TestCase): def test_client_catches_stream_not_found_exception(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamNotFoundException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass @@ -61,10 +61,10 @@ class FaviconClientTestCase(TestCase): def test_client_catches_stream_denied_exception(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamDeniedException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass @@ -76,10 +76,10 @@ class FaviconClientTestCase(TestCase): def test_client_catches_stream_timed_out(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamTimeOutException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass diff --git a/src/newsreader/news/collection/tests/favicon/collector/tests.py b/src/newsreader/news/collection/tests/favicon/collector/tests.py index 44254a5..cb73a7c 100644 --- a/src/newsreader/news/collection/tests/favicon/collector/tests.py +++ b/src/newsreader/news/collection/tests/favicon/collector/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase @@ -38,8 +38,8 @@ class FaviconCollectorTestCase(TestCase): def test_simple(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] - self.mocked_website_read.return_value = (website_mock, MagicMock()) + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] + self.mocked_website_read.return_value = (website_mock, Mock(rule=rule)) collector = FaviconCollector() collector.collect() @@ -54,8 +54,11 @@ class FaviconCollectorTestCase(TestCase): def test_empty_stream(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] - self.mocked_website_read.return_value = (BeautifulSoup("", "lxml"), MagicMock()) + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] + self.mocked_website_read.return_value = ( + BeautifulSoup("", "lxml"), + Mock(rule=rule), + ) collector = FaviconCollector() collector.collect() @@ -70,7 +73,7 @@ class FaviconCollectorTestCase(TestCase): def test_not_found(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamNotFoundException collector = FaviconCollector() @@ -86,7 +89,7 @@ class FaviconCollectorTestCase(TestCase): def test_denied(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamDeniedException collector = FaviconCollector() @@ -102,7 +105,7 @@ class FaviconCollectorTestCase(TestCase): def test_forbidden(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamForbiddenException collector = FaviconCollector() @@ -118,7 +121,7 @@ class FaviconCollectorTestCase(TestCase): def test_timed_out(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamTimeOutException collector = FaviconCollector() @@ -134,7 +137,7 @@ class FaviconCollectorTestCase(TestCase): def test_wrong_stream_content_type(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamParseException collector = FaviconCollector() diff --git a/src/newsreader/news/collection/tests/feed/builder/tests.py b/src/newsreader/news/collection/tests/feed/builder/tests.py index 4a6eb69..571a7cd 100644 --- a/src/newsreader/news/collection/tests/feed/builder/tests.py +++ b/src/newsreader/news/collection/tests/feed/builder/tests.py @@ -1,5 +1,5 @@ from datetime import date, datetime, time -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase from django.utils import timezone @@ -24,9 +24,10 @@ class FeedBuilderTestCase(TestCase): def test_basic_entry(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -55,9 +56,10 @@ class FeedBuilderTestCase(TestCase): def test_multiple_entries(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((multiple_mock, mock_stream)) as builder: + with builder(multiple_mock, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -116,9 +118,10 @@ class FeedBuilderTestCase(TestCase): def test_entries_without_remote_identifier(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_identifier, mock_stream)) as builder: + with builder(mock_without_identifier, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -155,9 +158,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_publication_date(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_publish_date, mock_stream)) as builder: + with builder(mock_without_publish_date, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -187,9 +191,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_url(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_url, mock_stream)) as builder: + with builder(mock_without_url, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -213,9 +218,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_body(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_body, mock_stream)) as builder: + with builder(mock_without_body, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -247,9 +253,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_author(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_author, mock_stream)) as builder: + with builder(mock_without_author, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -275,9 +282,10 @@ class FeedBuilderTestCase(TestCase): def test_empty_entries(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_entries, mock_stream)) as builder: + with builder(mock_without_entries, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -285,7 +293,7 @@ class FeedBuilderTestCase(TestCase): def test_update_entries(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) existing_first_post = FeedPostFactory.create( remote_identifier="28f79ae4-8f9a-11e9-b143-00163ef6bee7", rule=rule @@ -295,7 +303,8 @@ class FeedBuilderTestCase(TestCase): remote_identifier="a5479c66-8fae-11e9-8422-00163ef6bee7", rule=rule ) - with builder((mock_with_update_entries, mock_stream)) as builder: + with builder(mock_with_update_entries, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 3) @@ -315,9 +324,10 @@ class FeedBuilderTestCase(TestCase): def test_html_sanitizing(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_html, mock_stream)) as builder: + with builder(mock_with_html, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -337,9 +347,10 @@ class FeedBuilderTestCase(TestCase): def test_long_author_text_is_truncated(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_long_author, mock_stream)) as builder: + with builder(mock_with_long_author, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -351,9 +362,10 @@ class FeedBuilderTestCase(TestCase): def test_long_title_text_is_truncated(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_long_title, mock_stream)) as builder: + with builder(mock_with_long_title, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -366,9 +378,10 @@ class FeedBuilderTestCase(TestCase): def test_long_title_exotic_title(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_long_exotic_title, mock_stream)) as builder: + with builder(mock_with_long_exotic_title, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -381,9 +394,10 @@ class FeedBuilderTestCase(TestCase): def test_content_detail_is_prioritized_if_longer(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_longer_content_detail, mock_stream)) as builder: + with builder(mock_with_longer_content_detail, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -398,9 +412,10 @@ class FeedBuilderTestCase(TestCase): def test_content_detail_is_not_prioritized_if_shorter(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_shorter_content_detail, mock_stream)) as builder: + with builder(mock_with_shorter_content_detail, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -414,9 +429,10 @@ class FeedBuilderTestCase(TestCase): def test_content_detail_is_concatinated(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_multiple_content_detail, mock_stream)) as builder: + with builder(mock_with_multiple_content_detail, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() diff --git a/src/newsreader/news/collection/tests/feed/client/tests.py b/src/newsreader/news/collection/tests/feed/client/tests.py index 24eb214..9a2365e 100644 --- a/src/newsreader/news/collection/tests/feed/client/tests.py +++ b/src/newsreader/news/collection/tests/feed/client/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase from django.utils.lorem_ipsum import words @@ -28,7 +28,7 @@ class FeedClientTestCase(TestCase): def test_client_retrieves_single_rules(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) self.mocked_read.return_value = (simple_mock, mock_stream) diff --git a/src/newsreader/news/collection/tests/feed/collector/tests.py b/src/newsreader/news/collection/tests/feed/collector/tests.py index 5a1bac1..a7f3573 100644 --- a/src/newsreader/news/collection/tests/feed/collector/tests.py +++ b/src/newsreader/news/collection/tests/feed/collector/tests.py @@ -1,6 +1,6 @@ from datetime import date, datetime, time from time import struct_time -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase from django.utils import timezone @@ -26,6 +26,7 @@ from newsreader.news.core.tests.factories import FeedPostFactory from .mocks import duplicate_mock, empty_mock, multiple_mock, multiple_update_mock +@freeze_time("2019-10-30 12:30:00") class FeedCollectorTestCase(TestCase): def setUp(self): self.maxDiff = None @@ -39,43 +40,42 @@ class FeedCollectorTestCase(TestCase): def tearDown(self): patch.stopall() - @freeze_time("2019-10-30 12:30:00") def test_simple_batch(self): self.mocked_parse.return_value = multiple_mock - rule = FeedFactory() + rule = FeedFactory() collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() self.assertEquals(Post.objects.count(), 3) self.assertEquals(rule.succeeded, True) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) self.assertEquals(rule.error, None) - @freeze_time("2019-10-30 12:30:00") def test_emtpy_batch(self): - self.mocked_fetch.return_value = MagicMock() + self.mocked_fetch.return_value = Mock() self.mocked_parse.return_value = empty_mock + rule = FeedFactory() collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, True) self.assertEquals(rule.error, None) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) def test_not_found(self): self.mocked_fetch.side_effect = StreamNotFoundException - rule = FeedFactory() + rule = FeedFactory() collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -85,58 +85,59 @@ class FeedCollectorTestCase(TestCase): def test_denied(self): self.mocked_fetch.side_effect = StreamDeniedException - last_suceeded = timezone.make_aware( - datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) - ) - rule = FeedFactory(last_suceeded=last_suceeded) + + old_run = timezone.make_aware(datetime(2019, 10, 30, 12, 30)) + rule = FeedFactory(last_run=old_run) collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, False) self.assertEquals(rule.error, "Stream does not have sufficient permissions") - self.assertEquals(rule.last_suceeded, last_suceeded) + self.assertEquals(rule.last_run, timezone.now()) def test_forbidden(self): self.mocked_fetch.side_effect = StreamForbiddenException - last_suceeded = timezone.make_aware( - datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) - ) - rule = FeedFactory(last_suceeded=last_suceeded) + + old_run = pytz.utc.localize(datetime(2019, 10, 30, 12, 30)) + rule = FeedFactory(last_run=old_run) collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, False) self.assertEquals(rule.error, "Stream forbidden") - self.assertEquals(rule.last_suceeded, last_suceeded) + self.assertEquals(rule.last_run, timezone.now()) def test_timed_out(self): self.mocked_fetch.side_effect = StreamTimeOutException - last_suceeded = timezone.make_aware( + + last_run = timezone.make_aware( datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) ) - rule = FeedFactory(last_suceeded=last_suceeded) + rule = FeedFactory(last_run=last_run) collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, False) self.assertEquals(rule.error, "Stream timed out") - self.assertEquals(rule.last_suceeded, last_suceeded) + self.assertEquals( + rule.last_run, pytz.utc.localize(datetime(2019, 10, 30, 12, 30)) + ) - @freeze_time("2019-10-30 12:30:00") def test_duplicates(self): self.mocked_parse.return_value = duplicate_mock + rule = FeedFactory() aware_datetime = build_publication_date( @@ -186,10 +187,9 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 3) self.assertEquals(rule.succeeded, True) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) self.assertEquals(rule.error, None) - @freeze_time("2019-02-22 12:30:00") def test_items_with_identifiers_get_updated(self): self.mocked_parse.return_value = multiple_update_mock rule = FeedFactory() @@ -231,7 +231,7 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 3) self.assertEquals(rule.succeeded, True) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) self.assertEquals(rule.error, None) self.assertEquals( @@ -245,23 +245,3 @@ class FeedCollectorTestCase(TestCase): self.assertEquals( third_post.title, "Birmingham head teacher threatened over LGBT lessons" ) - - @freeze_time("2019-02-22 12:30:00") - def test_disabled_rules(self): - rules = (FeedFactory(enabled=False), FeedFactory(enabled=True)) - - self.mocked_parse.return_value = multiple_mock - - collector = FeedCollector() - collector.collect() - - for rule in rules: - rule.refresh_from_db() - - self.assertEquals(Post.objects.count(), 3) - self.assertEquals(rules[1].succeeded, True) - self.assertEquals(rules[1].last_suceeded, timezone.now()) - self.assertEquals(rules[1].error, None) - - self.assertEquals(rules[0].last_suceeded, None) - self.assertEquals(rules[0].succeeded, False) diff --git a/src/newsreader/news/collection/tests/feed/stream/tests.py b/src/newsreader/news/collection/tests/feed/stream/tests.py index 82a09a3..f827c15 100644 --- a/src/newsreader/news/collection/tests/feed/stream/tests.py +++ b/src/newsreader/news/collection/tests/feed/stream/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase @@ -27,7 +27,7 @@ class FeedStreamTestCase(TestCase): patch.stopall() def test_simple_stream(self): - self.mocked_fetch.return_value = MagicMock(content=simple_mock) + self.mocked_fetch.return_value = Mock(content=simple_mock) rule = FeedFactory() stream = FeedStream(rule) @@ -95,7 +95,7 @@ class FeedStreamTestCase(TestCase): @patch("newsreader.news.collection.feed.parse") def test_stream_raises_parse_exception(self, mocked_parse): - self.mocked_fetch.return_value = MagicMock() + self.mocked_fetch.return_value = Mock() mocked_parse.side_effect = TypeError rule = FeedFactory() diff --git a/src/newsreader/news/collection/tests/reddit/builder/tests.py b/src/newsreader/news/collection/tests/reddit/builder/tests.py index 9c1a046..11cf549 100644 --- a/src/newsreader/news/collection/tests/reddit/builder/tests.py +++ b/src/newsreader/news/collection/tests/reddit/builder/tests.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase @@ -20,9 +20,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -65,9 +66,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((empty_mock, mock_stream)) as builder: + with builder(empty_mock, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -76,9 +78,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((unknown_mock, mock_stream)) as builder: + with builder(unknown_mock, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -95,9 +98,10 @@ class RedditBuilderTestCase(TestCase): ) builder = RedditBuilder - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -132,9 +136,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((unsanitized_mock, mock_stream)) as builder: + with builder(unsanitized_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -149,9 +154,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((author_mock, mock_stream)) as builder: + with builder(author_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -166,9 +172,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((title_mock, mock_stream)) as builder: + with builder(title_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -186,9 +193,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((duplicate_mock, mock_stream)) as builder: + with builder(duplicate_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -200,13 +208,14 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) duplicate_post = RedditPostFactory( remote_identifier="hm0qct", rule=subreddit, title="foo" ) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -231,9 +240,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((image_mock, mock_stream)) as builder: + with builder(image_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -262,9 +272,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((external_image_mock, mock_stream)) as builder: + with builder(external_image_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -302,9 +313,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((video_mock, mock_stream)) as builder: + with builder(video_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -328,9 +340,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((external_video_mock, mock_stream)) as builder: + with builder(external_video_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -354,9 +367,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((external_gifv_mock, mock_stream)) as builder: + with builder(external_gifv_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -376,9 +390,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get(remote_identifier="hngsj8") @@ -400,9 +415,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((unknown_mock, mock_stream)) as builder: + with builder(unknown_mock, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) diff --git a/src/newsreader/news/collection/tests/reddit/client/tests.py b/src/newsreader/news/collection/tests/reddit/client/tests.py index f2ee84d..4dcc10f 100644 --- a/src/newsreader/news/collection/tests/reddit/client/tests.py +++ b/src/newsreader/news/collection/tests/reddit/client/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from uuid import uuid4 from django.test import TestCase @@ -31,7 +31,7 @@ class RedditClientTestCase(TestCase): def test_client_retrieves_single_rules(self): subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) self.mocked_read.return_value = (simple_mock, mock_stream) @@ -150,7 +150,7 @@ class RedditClientTestCase(TestCase): def test_client_catches_long_exception_text(self): subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) self.mocked_read.side_effect = StreamParseException(message=words(1000)) diff --git a/src/newsreader/news/collection/tests/reddit/collector/tests.py b/src/newsreader/news/collection/tests/reddit/collector/tests.py index 1fd18b0..fa2f5d4 100644 --- a/src/newsreader/news/collection/tests/reddit/collector/tests.py +++ b/src/newsreader/news/collection/tests/reddit/collector/tests.py @@ -74,7 +74,7 @@ class RedditCollectorTestCase(TestCase): for subreddit in rules: with self.subTest(subreddit=subreddit): self.assertEquals(subreddit.succeeded, True) - self.assertEquals(subreddit.last_suceeded, timezone.now()) + self.assertEquals(subreddit.last_run, timezone.now()) self.assertEquals(subreddit.error, None) post = Post.objects.get( @@ -133,7 +133,7 @@ class RedditCollectorTestCase(TestCase): for subreddit in rules: with self.subTest(subreddit=subreddit): self.assertEquals(subreddit.succeeded, True) - self.assertEquals(subreddit.last_suceeded, timezone.now()) + self.assertEquals(subreddit.last_run, timezone.now()) self.assertEquals(subreddit.error, None) def test_not_found(self): diff --git a/src/newsreader/news/collection/tests/reddit/test_scheduler.py b/src/newsreader/news/collection/tests/reddit/test_scheduler.py index cd062b6..0f04d53 100644 --- a/src/newsreader/news/collection/tests/reddit/test_scheduler.py +++ b/src/newsreader/news/collection/tests/reddit/test_scheduler.py @@ -25,19 +25,19 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory( user=user_1, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=4), + last_run=timezone.now() - timedelta(days=4), enabled=True, ), CollectionRuleFactory( user=user_1, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=3), + last_run=timezone.now() - timedelta(days=3), enabled=True, ), CollectionRuleFactory( user=user_1, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=2), + last_run=timezone.now() - timedelta(days=2), enabled=True, ), ] @@ -46,19 +46,19 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory( user=user_2, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=4), + last_run=timezone.now() - timedelta(days=4), enabled=True, ), CollectionRuleFactory( user=user_2, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=3), + last_run=timezone.now() - timedelta(days=3), enabled=True, ), CollectionRuleFactory( user=user_2, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=2), + last_run=timezone.now() - timedelta(days=2), enabled=True, ), ] @@ -87,7 +87,7 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory.create_batch( name=f"rule-{index}", type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(seconds=index), + last_run=timezone.now() - timedelta(seconds=index), enabled=True, user=user, size=15, @@ -121,7 +121,7 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory( name=f"rule-{index}", type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(seconds=index), + last_run=timezone.now() - timedelta(seconds=index), enabled=True, user=user, ) diff --git a/src/newsreader/news/collection/tests/tests.py b/src/newsreader/news/collection/tests/tests.py index 363e0b5..c7f0bb0 100644 --- a/src/newsreader/news/collection/tests/tests.py +++ b/src/newsreader/news/collection/tests/tests.py @@ -1,10 +1,9 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase from bs4 import BeautifulSoup -from newsreader.news.collection.base import URLBuilder, WebsiteStream from newsreader.news.collection.exceptions import ( StreamDeniedException, StreamException, @@ -13,6 +12,7 @@ from newsreader.news.collection.exceptions import ( StreamParseException, StreamTimeOutException, ) +from newsreader.news.collection.favicon import WebsiteStream, WebsiteURLBuilder from newsreader.news.collection.tests.factories import CollectionRuleFactory from .mocks import feed_mock_without_link, simple_feed_mock, simple_mock @@ -20,117 +20,125 @@ from .mocks import feed_mock_without_link, simple_feed_mock, simple_mock class WebsiteStreamTestCase(TestCase): def setUp(self): - self.patched_fetch = patch("newsreader.news.collection.base.fetch") + self.patched_fetch = patch("newsreader.news.collection.favicon.fetch") self.mocked_fetch = self.patched_fetch.start() def tearDown(self): patch.stopall() def test_simple(self): - self.mocked_fetch.return_value = MagicMock(content=simple_mock) + self.mocked_fetch.return_value = Mock(content=simple_mock) - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) return_value = stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) - self.assertEquals(return_value, (BeautifulSoup(simple_mock, "lxml"), stream)) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") + self.assertEquals( + return_value, (BeautifulSoup(simple_mock, features="lxml"), stream) + ) def test_raises_exception(self): self.mocked_fetch.side_effect = StreamException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_raises_denied_exception(self): self.mocked_fetch.side_effect = StreamDeniedException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamDeniedException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_raises_stream_not_found_exception(self): self.mocked_fetch.side_effect = StreamNotFoundException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamNotFoundException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_stream_raises_time_out_exception(self): self.mocked_fetch.side_effect = StreamTimeOutException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamTimeOutException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_stream_raises_forbidden_exception(self): self.mocked_fetch.side_effect = StreamForbiddenException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamForbiddenException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") - @patch("newsreader.news.collection.base.WebsiteStream.parse") + @patch("newsreader.news.collection.favicon.WebsiteStream.parse") def test_stream_raises_parse_exception(self, mocked_parse): - self.mocked_fetch.return_value = MagicMock() + self.mocked_fetch.return_value = Mock() mocked_parse.side_effect = StreamParseException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamParseException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") -class URLBuilderTestCase(TestCase): +class WebsiteURLBuilderTestCase(TestCase): def test_simple(self): initial_rule = CollectionRuleFactory() - with URLBuilder((simple_feed_mock, MagicMock(rule=initial_rule))) as builder: - rule, url = builder.build() + with WebsiteURLBuilder(simple_feed_mock, Mock(rule=initial_rule)) as builder: + builder.build() + builder.save() - self.assertEquals(rule.pk, initial_rule.pk) - self.assertEquals(url, "https://www.bbc.co.uk/news/") + initial_rule.refresh_from_db() + + self.assertEquals(initial_rule.website_url, "https://www.bbc.co.uk/news/") def test_no_link(self): - initial_rule = CollectionRuleFactory() + initial_rule = CollectionRuleFactory(website_url=None) - with URLBuilder( - (feed_mock_without_link, MagicMock(rule=initial_rule)) + with WebsiteURLBuilder( + feed_mock_without_link, Mock(rule=initial_rule) ) as builder: - rule, url = builder.build() + builder.build() + builder.save() - self.assertEquals(rule.pk, initial_rule.pk) - self.assertEquals(url, None) + initial_rule.refresh_from_db() + + self.assertEquals(initial_rule.website_url, None) def test_no_data(self): - initial_rule = CollectionRuleFactory() + initial_rule = CollectionRuleFactory(website_url=None) - with URLBuilder((None, MagicMock(rule=initial_rule))) as builder: - rule, url = builder.build() + with WebsiteURLBuilder(None, Mock(rule=initial_rule)) as builder: + builder.build() + builder.save() - self.assertEquals(rule.pk, initial_rule.pk) - self.assertEquals(url, None) + initial_rule.refresh_from_db() + + self.assertEquals(initial_rule.website_url, None) diff --git a/src/newsreader/news/collection/tests/twitter/__init__.py b/src/newsreader/news/collection/tests/twitter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/builder/__init__.py b/src/newsreader/news/collection/tests/twitter/builder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py new file mode 100644 index 0000000..b330f2f --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -0,0 +1,2187 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=twitterapi&tweet_mode=extended" | python3 -m json.tool --sort-keys +# +# see https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/tweet-object +# and https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/extended-entities-object +# for more information about tweet objects + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Aug 07 00:17:05 +0000 2020", + "display_text_range": [11, 59], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "youtu.be/rDy7tPf6CT8", + "expanded_url": "https://youtu.be/rDy7tPf6CT8", + "indices": [36, 59], + "url": "https://t.co/trAcIxBMlX", + } + ], + "user_mentions": [ + { + "id": 975844884606275587, + "id_str": "975844884606275587", + "indices": [0, 10], + "name": "ArieNeo", + "screen_name": "ArieNeoSC", + } + ], + }, + "favorite_count": 19, + "favorited": False, + "full_text": "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX", + "geo": None, + "id": 1291528756373286914, + "id_str": "1291528756373286914", + "in_reply_to_screen_name": "ArieNeoSC", + "in_reply_to_status_id": 1291507356313038850, + "in_reply_to_status_id_str": "1291507356313038850", + "in_reply_to_user_id": 975844884606275587, + "in_reply_to_user_id_str": "975844884606275587", + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 5, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 29 19:01:47 +0000 2020", + "display_text_range": [10, 98], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 435221600, + "id_str": "435221600", + "indices": [0, 9], + "name": "Christopher Blough", + "screen_name": "RelicCcb", + } + ], + }, + "favorite_count": 1, + "favorited": False, + "full_text": "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing.", + "geo": None, + "id": 1288550304095416320, + "id_str": "1288550304095416320", + "in_reply_to_screen_name": "RelicCcb", + "in_reply_to_status_id": 1288475147951898625, + "in_reply_to_status_id_str": "1288475147951898625", + "in_reply_to_user_id": 435221600, + "in_reply_to_user_id_str": "435221600", + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 0, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +# contains tweets with "extended_entities" keys which contains native media objects +# which are in this case of the type "photo" +image_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jun 05 22:51:46 +0000 2020", + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/VjEeDrL1iA", + "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1", + "id": 1269039233072689152, + "id_str": "1269039233072689152", + "indices": [2, 25], + "media_url": "http://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", + "media_url_https": "https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", + "sizes": { + "large": {"h": 1073, "resize": "fit", "w": 1125}, + "medium": {"h": 1073, "resize": "fit", "w": 1125}, + "small": {"h": 649, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/VjEeDrL1iA", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/VjEeDrL1iA", + "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1", + "id": 1269039233072689152, + "id_str": "1269039233072689152", + "indices": [2, 25], + "media_url": "http://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", + "media_url_https": "https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", + "sizes": { + "large": {"h": 1073, "resize": "fit", "w": 1125}, + "medium": {"h": 1073, "resize": "fit", "w": 1125}, + "small": {"h": 649, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/VjEeDrL1iA", + }, + { + "display_url": "pic.twitter.com/VjEeDrL1iA", + "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1", + "id": 1269039233068527618, + "id_str": "1269039233068527618", + "indices": [2, 25], + "media_url": "http://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg", + "media_url_https": "https://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg", + "sizes": { + "large": {"h": 992, "resize": "fit", "w": 1472}, + "medium": {"h": 809, "resize": "fit", "w": 1200}, + "small": {"h": 458, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/VjEeDrL1iA", + }, + ] + }, + "favorite_count": 2139, + "favorited": False, + "geo": None, + "id": 1269039237166321664, + "id_str": "1269039237166321664", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "und", + "place": None, + "possibly_sensitive": False, + "possibly_sensitive_appealable": False, + "retweet_count": 427, + "retweeted": False, + "source": 'Twitter for iPhone', + "full_text": "_ https://t.co/VjEeDrL1iA", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Tue Nov 14 19:00:00 +0000 2017", + "default_profile": False, + "default_profile_image": False, + "description": "Grammy\u00ae Award Winning Beatmakr. https://t.co/SN23ei3EeC https://t.co/EkGRhZ1Bw9 https://t.co/eEb4NOmJLo", + "entities": { + "description": { + "urls": [ + { + "display_url": "soundcloud.com/knxwledge", + "expanded_url": "http://soundcloud.com/knxwledge", + "indices": [32, 55], + "url": "https://t.co/SN23ei3EeC", + }, + { + "display_url": "knxwledge.bandcamp.com", + "expanded_url": "http://knxwledge.bandcamp.com", + "indices": [56, 79], + "url": "https://t.co/EkGRhZ1Bw9", + }, + { + "display_url": "twitch.tv/knxwledge", + "expanded_url": "http://twitch.tv/knxwledge", + "indices": [80, 103], + "url": "https://t.co/eEb4NOmJLo", + }, + ] + }, + "url": { + "urls": [ + { + "display_url": "instagram.com/knxwledge/?hl=\u2026", + "expanded_url": "https://www.instagram.com/knxwledge/?hl=en", + "indices": [0, 23], + "url": "https://t.co/UcMYfiQXLx", + } + ] + }, + }, + "favourites_count": 363, + "follow_request_sent": None, + "followers_count": 31194, + "following": None, + "friends_count": 15, + "geo_enabled": False, + "has_extended_profile": False, + "id": 930510644763287552, + "id_str": "930510644763287552", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 56, + "location": "", + "name": "knxwledge", + "notifications": None, + "profile_background_color": "000000", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_image_url": "http://pbs.twimg.com/profile_images/1274913160898592768/jFi4VDtJ_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1274913160898592768/jFi4VDtJ_normal.jpg", + "profile_link_color": "ABB8C2", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "000000", + "profile_text_color": "000000", + "profile_use_background_image": False, + "protected": False, + "screen_name": "knxwledge", + "statuses_count": 713, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/UcMYfiQXLx", + "utc_offset": None, + "verified": False, + }, + } +] + +# contains tweets with "extended_entities" keys which contains native media objects +# which are in this case of the type "video" +video_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 18:36:00 +0000 2020", + "display_text_range": [0, 196], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/mZ8CAuq3SH", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/greycatroc", + "expanded_url": "http://robertsspaceindustries.com/greycatroc", + "indices": [173, 196], + "url": "https://t.co/2aH7qdOfSk", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "additional_media_info": { + "description": "", + "embeddable": True, + "monetizable": False, + "title": "", + }, + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/mZ8CAuq3SH", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 82967, + "variants": [ + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/pl/kMYgFEoRyoW99o-i.m3u8?tag=13", + }, + { + "bitrate": 2176000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/vid/1280x720/J05_p6q74ZUN4csg.mp4?tag=13", + }, + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/vid/640x360/ya3fVKeRdBs3cOoF.mp4?tag=13", + }, + { + "bitrate": 288000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/vid/480x270/WQkAozOts-hRoU1I.mp4?tag=13", + }, + ], + }, + } + ] + }, + "favorite_count": 289, + "favorited": False, + "full_text": "Small enough to access hard-to-reach ore deposits, but with enough power to get through the tough jobs, Greycat\u2019s ROC perfectly complements any mining operation. \n\nDetails: https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH", + "geo": None, + "id": 1291080532361527296, + "id_str": "1291080532361527296", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 64, + "retweeted": False, + "source": 'Twitter Media Studio', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 18:31:27 +0000 2020", + "display_text_range": [0, 213], + "entities": { + "hashtags": [{"indices": [157, 169], "text": "StarCitizen"}], + "media": [ + { + "display_url": "pic.twitter.com/lri5QijMoA", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291079386821582849/video/1", + "id": 1291070740347813889, + "id_str": "1291070740347813889", + "indices": [214, 237], + "media_url": "http://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/lri5QijMoA", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/comm-link/tran\u2026", + "expanded_url": "https://robertsspaceindustries.com/comm-link/transmission/17648-Alpha-310-Flight-Fight", + "indices": [190, 213], + "url": "https://t.co/6jT1yuZMiR", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "additional_media_info": { + "description": "", + "embeddable": True, + "monetizable": False, + "title": "", + }, + "display_url": "pic.twitter.com/lri5QijMoA", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291079386821582849/video/1", + "id": 1291070740347813889, + "id_str": "1291070740347813889", + "indices": [214, 237], + "media_url": "http://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/lri5QijMoA", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 83633, + "variants": [ + { + "bitrate": 288000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/vid/480x270/oGdSeLr5QQ-XcTns.mp4?tag=13", + }, + { + "bitrate": 2176000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/vid/1280x720/bql0evKsgYZhGPNP.mp4?tag=13", + }, + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/vid/640x360/lSL6mqB53HnwrUo4.mp4?tag=13", + }, + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/pl/_jJ-AYWSMr8ZS1WP.m3u8?tag=13", + }, + ], + }, + } + ] + }, + "favorite_count": 429, + "favorited": False, + "full_text": "Harness the power of improved high-speed dynamic combat. Feel the thrill of atmospheric flight like never before. Alpha 3.10 will change the way you play. \ud83d\ude80 #StarCitizen\n\nGet in the 'verse: https://t.co/6jT1yuZMiR https://t.co/lri5QijMoA", + "geo": None, + "id": 1291079386821582849, + "id_str": "1291079386821582849", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 117, + "retweeted": False, + "source": 'Twitter Media Studio', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +video_without_bitrate_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 18:36:00 +0000 2020", + "display_text_range": [0, 196], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/mZ8CAuq3SH", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/greycatroc", + "expanded_url": "http://robertsspaceindustries.com/greycatroc", + "indices": [173, 196], + "url": "https://t.co/2aH7qdOfSk", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "additional_media_info": { + "description": "", + "embeddable": True, + "monetizable": False, + "title": "", + }, + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/mZ8CAuq3SH", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 82967, + "variants": [ + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/pl/kMYgFEoRyoW99o-i.m3u8?tag=13", + } + ], + }, + } + ] + }, + "favorite_count": 289, + "favorited": False, + "full_text": "Small enough to access hard-to-reach ore deposits, but with enough power to get through the tough jobs, Greycat\u2019s ROC perfectly complements any mining operation. \n\nDetails: https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH", + "geo": None, + "id": 1291080532361527296, + "id_str": "1291080532361527296", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 64, + "retweeted": False, + "source": 'Twitter Media Studio', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + } +] + +# contains tweets with a "retweeted_status" key containing the retweeted tweet. +# the "retweet" cannot add hashtags, URLs or other details, see https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/entities-object#retweets-quote +retweet_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 21:01:02 +0000 2020", + "display_text_range": [0, 140], + "entities": { + "hashtags": [{"indices": [27, 39], "text": "StarCitizen"}], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 859293278100914176, + "id_str": "859293278100914176", + "indices": [3, 14], + "name": "Aleksandr Belov", + "screen_name": "Narayan_N7", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "full_text": "RT @Narayan_N7: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPle\u2026", + "geo": None, + "id": 1291117030486106112, + "id_str": "1291117030486106112", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 26, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 18:15:34 +0000 2020", + "display_text_range": [0, 250], + "entities": { + "hashtags": [{"indices": [11, 23], "text": "StarCitizen"}], + "symbols": [], + "urls": [ + { + "display_url": "youtu.be/aXXGnCbEas0", + "expanded_url": "https://youtu.be/aXXGnCbEas0", + "indices": [227, 250], + "url": "https://t.co/j4QahHzbw4", + } + ], + "user_mentions": [ + { + "id": 803542770, + "id_str": "803542770", + "indices": [193, 209], + "name": "Star Citizen", + "screen_name": "RobertsSpaceInd", + }, + { + "id": 803697073, + "id_str": "803697073", + "indices": [211, 225], + "name": "Cloud Imperium Games", + "screen_name": "CloudImperium", + }, + ], + }, + "favorite_count": 97, + "favorited": False, + "full_text": "New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease, share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe! \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\nhttps://t.co/j4QahHzbw4", + "geo": None, + "id": 1291075388798533633, + "id_str": "1291075388798533633", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 26, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Tue May 02 06:27:37 +0000 2017", + "default_profile": True, + "default_profile_image": False, + "description": "Enlist to Star Citizen: https://t.co/JOei50wjGK Content creator. #IWantToWorkAtCIG \n#StarCitizen #video #youtube #flickr #4K #panorama", + "entities": { + "description": { + "urls": [ + { + "display_url": "goo.gl/8CbEZm", + "expanded_url": "http://goo.gl/8CbEZm", + "indices": [24, 47], + "url": "https://t.co/JOei50wjGK", + } + ] + }, + "url": { + "urls": [ + { + "display_url": "youtube.com/user/sashaMOHC\u2026", + "expanded_url": "https://www.youtube.com/user/sashaMOHCTPwhite", + "indices": [0, 23], + "url": "https://t.co/ise14uN9Ja", + } + ] + }, + }, + "favourites_count": 1882, + "follow_request_sent": None, + "followers_count": 489, + "following": None, + "friends_count": 80, + "geo_enabled": True, + "has_extended_profile": True, + "id": 859293278100914176, + "id_str": "859293278100914176", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 16, + "location": "\u0421\u0430\u043d\u043a\u0442-\u041f\u0435\u0442\u0435\u0440\u0431\u0443\u0440\u0433, \u0420\u043e\u0441\u0441\u0438\u044f", + "name": "Aleksandr Belov", + "notifications": None, + "profile_background_color": "F5F8FA", + "profile_background_image_url": None, + "profile_background_image_url_https": None, + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/859293278100914176/1576841460", + "profile_image_url": "http://pbs.twimg.com/profile_images/1203066581573607425/5TEkxVJ3_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1203066581573607425/5TEkxVJ3_normal.jpg", + "profile_link_color": "1DA1F2", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "Narayan_N7", + "statuses_count": 1283, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/ise14uN9Ja", + "utc_offset": None, + "verified": False, + }, + }, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Thu Jul 30 13:15:25 +0000 2020", + "display_text_range": [0, 140], + "entities": { + "hashtags": [{"indices": [24, 40], "text": "CountdownToMars"}], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 11348282, + "id_str": "11348282", + "indices": [3, 8], + "name": "NASA", + "screen_name": "NASA", + }, + { + "id": 1232783237623119872, + "id_str": "1232783237623119872", + "indices": [123, 137], + "name": "NASA's Perseverance Mars Rover", + "screen_name": "NASAPersevere", + }, + ], + }, + "favorite_count": 0, + "favorited": False, + "full_text": "RT @NASA: LIVE NOW: The #CountdownToMars begins. \n\nWe are launching a historic mission to the Red Planet. Tune in to watch @NASAPersevere l\u2026", + "geo": None, + "id": 1288825524878336000, + "id_str": "1288825524878336000", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 8867, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Thu Jul 30 11:01:06 +0000 2020", + "display_text_range": [0, 236], + "entities": { + "hashtags": [{"indices": [14, 30], "text": "CountdownToMars"}], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/broadcasts/1\u2026", + "expanded_url": "https://twitter.com/i/broadcasts/1RDGlrkoEzNxL", + "indices": [213, 236], + "url": "https://t.co/JxyRCol01i", + } + ], + "user_mentions": [ + { + "id": 1232783237623119872, + "id_str": "1232783237623119872", + "indices": [113, 127], + "name": "NASA's Perseverance Mars Rover", + "screen_name": "NASAPersevere", + } + ], + }, + "favorite_count": 18327, + "favorited": False, + "full_text": "LIVE NOW: The #CountdownToMars begins. \n\nWe are launching a historic mission to the Red Planet. Tune in to watch @NASAPersevere liftoff and begin her mission to search for signs of ancient life on another world: https://t.co/JxyRCol01i", + "geo": None, + "id": 1288791726165983233, + "id_str": "1288791726165983233", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 8867, + "retweeted": False, + "source": 'Twitter Media Studio', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Dec 19 20:20:32 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Explore the universe and our home planet with NASA \ud83c\udf0e We usually post in EDT.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "nasa.gov", + "expanded_url": "http://www.nasa.gov/", + "indices": [0, 23], + "url": "https://t.co/HMJJbimQpV", + } + ] + }, + }, + "favourites_count": 11658, + "follow_request_sent": None, + "followers_count": 39440029, + "following": None, + "friends_count": 222, + "geo_enabled": False, + "has_extended_profile": True, + "id": 11348282, + "id_str": "11348282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 92535, + "location": "", + "name": "NASA", + "notifications": None, + "profile_background_color": "000000", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/11348282/1596217000", + "profile_image_url": "http://pbs.twimg.com/profile_images/1091070803184177153/TI2qItoi_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1091070803184177153/TI2qItoi_normal.jpg", + "profile_link_color": "205BA7", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "F3F2F2", + "profile_text_color": "000000", + "profile_use_background_image": True, + "protected": False, + "screen_name": "NASA", + "statuses_count": 61920, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/HMJJbimQpV", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter for iPhone', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +# contains tweets with a "quoted_status" key containing the quoted tweet. +# quoted tweets can add hashtags, URL's and other details as it adds content "on top" of the quoted tweet see https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/entities-object#retweets-quotes +quoted_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 00:05:24 +0000 2020", + "display_text_range": [0, 13], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/hugolisoir/sta\u2026", + "expanded_url": "https://twitter.com/hugolisoir/status/1290778178793897992", + "indices": [14, 37], + "url": "https://t.co/WyznJwCJLp", + } + ], + "user_mentions": [], + }, + "favorite_count": 576, + "favorited": False, + "full_text": "Bonne nuit \ud83c\udf3a\ud83d\udeeb https://t.co/WyznJwCJLp", + "geo": None, + "id": 1290801039075979264, + "id_str": "1290801039075979264", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "fr", + "place": None, + "possibly_sensitive": False, + "quoted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Aug 04 22:34:33 +0000 2020", + "display_text_range": [0, 57], + "entities": { + "hashtags": [{"indices": [0, 12], "text": "Starcitizen"}], + "media": [ + { + "display_url": "pic.twitter.com/xCXun68V3r", + "expanded_url": "https://twitter.com/hugolisoir/status/1290778178793897992/video/1", + "id": 1290778053623382017, + "id_str": "1290778053623382017", + "indices": [58, 81], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/xCXun68V3r", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 803542770, + "id_str": "803542770", + "indices": [41, 57], + "name": "Star Citizen", + "screen_name": "RobertsSpaceInd", + } + ], + }, + "extended_entities": { + "media": [ + { + "additional_media_info": {"monetizable": False}, + "display_url": "pic.twitter.com/xCXun68V3r", + "expanded_url": "https://twitter.com/hugolisoir/status/1290778178793897992/video/1", + "id": 1290778053623382017, + "id_str": "1290778053623382017", + "indices": [58, 81], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/xCXun68V3r", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 39901, + "variants": [ + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/vid/640x360/jYjO0H2SYSycTi-e.mp4?tag=10", + }, + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/pl/wFnVMLjVWi7OKy2o.m3u8?tag=10", + }, + { + "bitrate": 2176000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/vid/1280x720/H-BXvYdM0AcSKXpk.mp4?tag=10", + }, + { + "bitrate": 256000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/vid/480x270/aWhSjP1gK7djKZUK.mp4?tag=10", + }, + ], + }, + } + ] + }, + "favorite_count": 400, + "favorited": False, + "full_text": "#Starcitizen Le jeu est beau. Bonne nuit @RobertsSpaceInd https://t.co/xCXun68V3r", + "geo": None, + "id": 1290778178793897992, + "id_str": "1290778178793897992", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "fr", + "place": None, + "possibly_sensitive": False, + "retweet_count": 76, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Tue Mar 22 12:00:36 +0000 2011", + "default_profile": False, + "default_profile_image": False, + "description": "Youtuber Partner / Twitch Partner / Membre du @CurryClub_CC\nInsta - hugolisoir\nParrain de @AbyssalProject", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "youtube.com/channel/UCDC6D\u2026", + "expanded_url": "https://www.youtube.com/channel/UCDC6DBi0kRp6Jk21xqfvFLA", + "indices": [0, 23], + "url": "https://t.co/p3CVR2I068", + } + ] + }, + }, + "favourites_count": 20935, + "follow_request_sent": None, + "followers_count": 23269, + "following": None, + "friends_count": 703, + "geo_enabled": True, + "has_extended_profile": False, + "id": 270320632, + "id_str": "270320632", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 116, + "location": "Nantes, France", + "name": "Hugo Lisoir #ZLAN2020", + "notifications": None, + "profile_background_color": "000000", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme15/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme15/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/270320632/1499086260", + "profile_image_url": "http://pbs.twimg.com/profile_images/1264841251305730048/vyUJVCvW_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1264841251305730048/vyUJVCvW_normal.jpg", + "profile_link_color": "ABB8C2", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "000000", + "profile_text_color": "000000", + "profile_use_background_image": False, + "protected": False, + "screen_name": "hugolisoir", + "statuses_count": 7507, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/p3CVR2I068", + "utc_offset": None, + "verified": False, + }, + }, + "quoted_status_id": 1290778178793897992, + "quoted_status_id_str": "1290778178793897992", + "quoted_status_permalink": { + "display": "twitter.com/hugolisoir/sta\u2026", + "expanded": "https://twitter.com/hugolisoir/status/1290778178793897992", + "url": "https://t.co/WyznJwCJLp", + }, + "retweet_count": 60, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jul 31 22:00:55 +0000 2020", + "display_text_range": [0, 32], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/UberFacts/stat\u2026", + "expanded_url": "https://twitter.com/UberFacts/status/1289273883493675009", + "indices": [33, 56], + "url": "https://t.co/LLPVr8oU7F", + } + ], + "user_mentions": [], + }, + "favorite_count": 263, + "favorited": False, + "full_text": "Here's to our lovely Avocados! \ud83d\udd79 https://t.co/LLPVr8oU7F", + "geo": None, + "id": 1289320160021495809, + "id_str": "1289320160021495809", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "quoted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jul 31 18:57:02 +0000 2020", + "display_text_range": [0, 34], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/8QRycx9QB2", + "expanded_url": "https://twitter.com/UberFacts/status/1289273883493675009/photo/1", + "id": 1289273880570363907, + "id_str": "1289273880570363907", + "indices": [35, 58], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "sizes": { + "large": {"h": 500, "resize": "fit", "w": 500}, + "medium": {"h": 500, "resize": "fit", "w": 500}, + "small": {"h": 500, "resize": "fit", "w": 500}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/8QRycx9QB2", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/8QRycx9QB2", + "expanded_url": "https://twitter.com/UberFacts/status/1289273883493675009/photo/1", + "id": 1289273880570363907, + "id_str": "1289273880570363907", + "indices": [35, 58], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "sizes": { + "large": {"h": 500, "resize": "fit", "w": 500}, + "medium": {"h": 500, "resize": "fit", "w": 500}, + "small": {"h": 500, "resize": "fit", "w": 500}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "animated_gif", + "url": "https://t.co/8QRycx9QB2", + "video_info": { + "aspect_ratio": [1, 1], + "variants": [ + { + "bitrate": 0, + "content_type": "video/mp4", + "url": "https://video.twimg.com/tweet_video/EeRrw3WWAAMKVF0.mp4", + } + ], + }, + } + ] + }, + "favorite_count": 1550, + "favorited": False, + "full_text": "July 31st is National Avocado Day! https://t.co/8QRycx9QB2", + "geo": None, + "id": 1289273883493675009, + "id_str": "1289273883493675009", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 380, + "retweeted": False, + "source": 'Buffer', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Sun Dec 06 16:07:01 +0000 2009", + "default_profile": False, + "default_profile_image": False, + "description": "The most unimportant things you'll never need to know.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "uber-facts.com", + "expanded_url": "http://uber-facts.com/", + "indices": [0, 23], + "url": "https://t.co/3ycpGqEL9n", + } + ] + }, + }, + "favourites_count": 1297, + "follow_request_sent": None, + "followers_count": 13810392, + "following": None, + "friends_count": 1, + "geo_enabled": True, + "has_extended_profile": False, + "id": 95023423, + "id_str": "95023423", + "is_translation_enabled": True, + "is_translator": False, + "lang": None, + "listed_count": 15141, + "location": "Worldwide!", + "name": "UberFacts", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/95023423/1587338728", + "profile_image_url": "http://pbs.twimg.com/profile_images/615696617165885440/JDbUuo9H_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/615696617165885440/JDbUuo9H_normal.jpg", + "profile_link_color": "0D9BA8", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "FFFFFF", + "profile_text_color": "000000", + "profile_use_background_image": True, + "protected": False, + "screen_name": "UberFacts", + "statuses_count": 202253, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ycpGqEL9n", + "utc_offset": None, + "verified": True, + }, + }, + "quoted_status_id": 1289273883493675009, + "quoted_status_id_str": "1289273883493675009", + "quoted_status_permalink": { + "display": "twitter.com/UberFacts/stat\u2026", + "expanded": "https://twitter.com/UberFacts/status/1289273883493675009", + "url": "https://t.co/LLPVr8oU7F", + }, + "retweet_count": 24, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +# contains tweets with "extended_entities" keys which contains native media objects +# which are in this case of the type "animated_gif" +gif_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jul 31 23:10:55 +0000 2020", + "display_text_range": [12, 12], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/wxvioLCJ6h", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1289337776140296193/photo/1", + "id": 1289337769521606656, + "id_str": "1289337769521606656", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "sizes": { + "large": {"h": 210, "resize": "fit", "w": 250}, + "medium": {"h": 210, "resize": "fit", "w": 250}, + "small": {"h": 210, "resize": "fit", "w": 250}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/wxvioLCJ6h", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 994361231057346561, + "id_str": "994361231057346561", + "indices": [0, 12], + "name": "Xenosystems", + "screen_name": "Xenosystems", + } + ], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/wxvioLCJ6h", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1289337776140296193/photo/1", + "id": 1289337769521606656, + "id_str": "1289337769521606656", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "sizes": { + "large": {"h": 210, "resize": "fit", "w": 250}, + "medium": {"h": 210, "resize": "fit", "w": 250}, + "small": {"h": 210, "resize": "fit", "w": 250}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "animated_gif", + "url": "https://t.co/wxvioLCJ6h", + "video_info": { + "aspect_ratio": [25, 21], + "variants": [ + { + "bitrate": 0, + "content_type": "video/mp4", + "url": "https://video.twimg.com/tweet_video/EeSl3sPUcAAyE4J.mp4", + } + ], + }, + } + ] + }, + "favorite_count": 13, + "favorited": False, + "full_text": "@Xenosystems https://t.co/wxvioLCJ6h", + "geo": None, + "id": 1289337776140296193, + "id_str": "1289337776140296193", + "in_reply_to_screen_name": "Xenosystems", + "in_reply_to_status_id": 1289324787815178242, + "in_reply_to_status_id_str": "1289324787815178242", + "in_reply_to_user_id": 994361231057346561, + "in_reply_to_user_id_str": "994361231057346561", + "is_quote_status": False, + "lang": "und", + "place": None, + "possibly_sensitive": False, + "retweet_count": 1, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Thu Jul 30 22:30:29 +0000 2020", + "display_text_range": [12, 12], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/DTbhK1pTc4", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1288965215648849920/photo/1", + "id": 1288965209596420097, + "id_str": "1288965209596420097", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "sizes": { + "large": {"h": 278, "resize": "fit", "w": 498}, + "medium": {"h": 278, "resize": "fit", "w": 498}, + "small": {"h": 278, "resize": "fit", "w": 498}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/DTbhK1pTc4", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 994361231057346561, + "id_str": "994361231057346561", + "indices": [0, 12], + "name": "Xenosystems", + "screen_name": "Xenosystems", + } + ], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/DTbhK1pTc4", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1288965215648849920/photo/1", + "id": 1288965209596420097, + "id_str": "1288965209596420097", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "sizes": { + "large": {"h": 278, "resize": "fit", "w": 498}, + "medium": {"h": 278, "resize": "fit", "w": 498}, + "small": {"h": 278, "resize": "fit", "w": 498}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "animated_gif", + "url": "https://t.co/DTbhK1pTc4", + "video_info": { + "aspect_ratio": [249, 139], + "variants": [ + { + "bitrate": 0, + "content_type": "video/mp4", + "url": "https://video.twimg.com/tweet_video/EeNTB2XU4AE-z5Y.mp4", + } + ], + }, + } + ] + }, + "favorite_count": 20, + "favorited": False, + "full_text": "@Xenosystems https://t.co/DTbhK1pTc4", + "geo": None, + "id": 1288965215648849920, + "id_str": "1288965215648849920", + "in_reply_to_screen_name": "Xenosystems", + "in_reply_to_status_id": 1288960722349719554, + "in_reply_to_status_id_str": "1288960722349719554", + "in_reply_to_user_id": 994361231057346561, + "in_reply_to_user_id_str": "994361231057346561", + "is_quote_status": False, + "lang": "und", + "place": None, + "possibly_sensitive": False, + "retweet_count": 0, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +unsanitized_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Aug 07 00:17:05 +0000 2020", + "display_text_range": [11, 59], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "youtu.be/rDy7tPf6CT8", + "expanded_url": "https://youtu.be/rDy7tPf6CT8", + "indices": [36, 59], + "url": "https://t.co/trAcIxBMlX", + } + ], + "user_mentions": [ + { + "id": 975844884606275587, + "id_str": "975844884606275587", + "indices": [0, 10], + "name": "ArieNeo", + "screen_name": "ArieNeoSC", + } + ], + }, + "favorite_count": 19, + "favorited": False, + "full_text": "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX
    ", + "geo": None, + "id": 1291528756373286914, + "id_str": "1291528756373286914", + "in_reply_to_screen_name": "ArieNeoSC", + "in_reply_to_status_id": 1291507356313038850, + "in_reply_to_status_id_str": "1291507356313038850", + "in_reply_to_user_id": 975844884606275587, + "in_reply_to_user_id_str": "975844884606275587", + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 5, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + } +] diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py new file mode 100644 index 0000000..37d7ad7 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -0,0 +1,412 @@ +from datetime import datetime +from unittest.mock import Mock + +from django.test import TestCase +from django.utils.safestring import mark_safe + +import pytz + +from ftfy import fix_text + +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.builder.mocks import ( + gif_mock, + image_mock, + quoted_mock, + retweet_mock, + simple_mock, + unsanitized_mock, + video_mock, + video_without_bitrate_mock, +) +from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder +from newsreader.news.collection.utils import truncate_text +from newsreader.news.core.models import Post +from newsreader.news.core.tests.factories import PostFactory + + +class TwitterBuilderTestCase(TestCase): + def setUp(self): + self.maxDiff = None + + def test_simple_post(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(simple_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291528756373286914", "1288550304095416320"), posts.keys() + ) + + post = posts["1291528756373286914"] + + full_text = ( + "@ArieNeoSC Here you go, goodnight!\n\n" + """https://t.co/trAcIxBMlX""" + ) + + self.assertEquals(post.rule, profile) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX", + ), + ) + self.assertEquals(post.body, mark_safe(full_text)) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1291528756373286914" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 8, 7, 0, 17, 5)) + ) + + post = posts["1288550304095416320"] + + full_text = "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing." + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, truncate_text(Post, "title", full_text)) + self.assertEquals(post.body, mark_safe(full_text)) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1288550304095416320" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 7, 29, 19, 1, 47)) + ) + + # note that only one media type can be uploaded to an Tweet + # see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object + def test_images_in_post(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(image_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual(("1269039237166321664",), posts.keys()) + + post = posts["1269039237166321664"] + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, "_ https://t.co/VjEeDrL1iA") + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1269039237166321664" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 6, 5, 22, 51, 46)) + ) + + self.assertInHTML( + """https://t.co/VjEeDrL1iA""", + post.body, + count=1, + ) + self.assertInHTML( + """
    1269039233072689152
    """, + post.body, + count=1, + ) + self.assertInHTML( + """
    1269039233068527618
    """, + post.body, + count=1, + ) + + def test_videos_in_post(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(video_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291080532361527296", "1291079386821582849"), posts.keys() + ) + + post = posts["1291080532361527296"] + + full_text = fix_text( + "Small enough to access hard-to-reach ore deposits, but with enough" + " power to get through the tough jobs, Greycat\u2019s ROC perfectly" + " complements any mining operation. \n\nDetails:" + """ https://t.co/2aH7qdOfSk""" + """ https://t.co/mZ8CAuq3SH""" + ) + + self.assertEquals(post.rule, profile) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + fix_text( + "Small enough to access hard-to-reach ore deposits, but with enough" + " power to get through the tough jobs, Greycat\u2019s ROC perfectly" + " complements any mining operation. \n\nDetails:" + " https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH" + ), + ), + ) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1291080532361527296" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 8, 5, 18, 36, 0)) + ) + + self.assertIn(full_text, post.body) + self.assertInHTML( + """
    """, + post.body, + count=1, + ) + + def test_video_without_bitrate(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(video_without_bitrate_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual(("1291080532361527296",), posts.keys()) + + post = posts["1291080532361527296"] + + self.assertInHTML( + """
    """, + post.body, + count=1, + ) + + def test_GIFs_in_post(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(gif_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1289337776140296193", "1288965215648849920"), posts.keys() + ) + + post = posts["1289337776140296193"] + + self.assertInHTML( + """
    """, + post.body, + count=1, + ) + + self.assertIn( + """@Xenosystems https://t.co/wxvioLCJ6h""", + post.body, + ) + + def test_retweet_post(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(retweet_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291117030486106112", "1288825524878336000"), posts.keys() + ) + + post = posts["1291117030486106112"] + + self.assertIn( + fix_text( + "RT @Narayan_N7: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo," + " the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPle\u2026" + ), + post.body, + ) + + self.assertIn( + fix_text( + "Original tweet: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch" + " 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease," + " share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe!" + " \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\n" + """https://t.co/j4QahHzbw4""" + ), + post.body, + ) + + def test_quoted_post(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(quoted_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1290801039075979264", "1289320160021495809"), posts.keys() + ) + + post = posts["1290801039075979264"] + + self.assertIn( + fix_text( + "Bonne nuit \ud83c\udf3a\ud83d\udeeb" + """ https://t.co/WyznJwCJLp""" + ), + post.body, + ) + + self.assertIn( + fix_text( + "Quoted tweet: #Starcitizen Le jeu est beau. Bonne nuit" + """ @RobertsSpaceInd https://t.co/xCXun68V3r""" + ), + post.body, + ) + + def test_empty_data(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder([], mock_stream) as builder: + builder.build() + builder.save() + + self.assertEquals(Post.objects.count(), 0) + + def test_html_sanitizing(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(unsanitized_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual(("1291528756373286914",), posts.keys()) + + post = posts["1291528756373286914"] + + full_text = ( + "@ArieNeoSC Here you go, goodnight!\n\n" + """https://t.co/trAcIxBMlX""" + "
    " + ) + + self.assertEquals(post.rule, profile) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" + "
    ", + ), + ) + self.assertEquals(post.body, mark_safe(full_text)) + + self.assertInHTML("", post.body, count=0) + self.assertInHTML("
    ", post.body, count=1) + + self.assertInHTML("", post.title, count=0) + self.assertInHTML("
    ", post.title, count=1) + + def test_urlize_on_urls(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(simple_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291528756373286914", "1288550304095416320"), posts.keys() + ) + + post = posts["1291528756373286914"] + + full_text = ( + "@ArieNeoSC Here you go, goodnight!\n\n" + """https://t.co/trAcIxBMlX""" + ) + + self.assertEquals(post.rule, profile) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX", + ), + ) + self.assertEquals(post.body, mark_safe(full_text)) + + def test_existing_posts(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + PostFactory(rule=profile, remote_identifier="1291528756373286914") + PostFactory(rule=profile, remote_identifier="1288550304095416320") + + with builder(simple_mock, mock_stream) as builder: + builder.build() + builder.save() + + self.assertEquals(Post.objects.count(), 2) diff --git a/src/newsreader/news/collection/tests/twitter/client/__init__.py b/src/newsreader/news/collection/tests/twitter/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/client/mocks.py b/src/newsreader/news/collection/tests/twitter/client/mocks.py new file mode 100644 index 0000000..1b7c6a2 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/client/mocks.py @@ -0,0 +1,225 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] diff --git a/src/newsreader/news/collection/tests/twitter/client/tests.py b/src/newsreader/news/collection/tests/twitter/client/tests.py new file mode 100644 index 0000000..387ffef --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/client/tests.py @@ -0,0 +1,162 @@ +from unittest.mock import Mock, patch +from uuid import uuid4 + +from django.test import TestCase +from django.utils.lorem_ipsum import words + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamNotFoundException, + StreamParseException, + StreamTimeOutException, + StreamTooManyException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.twitter import TwitterClient + +from .mocks import simple_mock + + +class TwitterClientTestCase(TestCase): + def setUp(self): + patched_read = patch("newsreader.news.collection.twitter.TwitterStream.read") + self.mocked_read = patched_read.start() + + def tearDown(self): + patch.stopall() + + def test_simple(self): + timeline = TwitterTimelineFactory() + mock_stream = Mock(rule=timeline) + + self.mocked_read.return_value = (simple_mock, mock_stream) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertEquals(data, simple_mock) + self.assertEquals(stream, mock_stream) + + self.mocked_read.assert_called() + + def test_client_catches_stream_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamException(message="Stream exception") + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream exception") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_not_found_exception(self): + timeline = TwitterTimelineFactory.create() + + self.mocked_read.side_effect = StreamNotFoundException( + message="Stream not found" + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream not found") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_denied_exception(self): + user = UserFactory( + twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4()) + ) + timeline = TwitterTimelineFactory(user=user) + + self.mocked_read.side_effect = StreamDeniedException(message="Token expired") + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Token expired") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + user.refresh_from_db() + timeline.refresh_from_db() + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) + + def test_client_catches_stream_timed_out_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamTimeOutException( + message="Stream timed out" + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream timed out") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_too_many_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamTooManyException + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Too many requests") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_parse_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamParseException( + message="Stream could not be parsed" + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream could not be parsed") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_long_exception_text(self): + timeline = TwitterTimelineFactory() + mock_stream = Mock(rule=timeline) + + self.mocked_read.side_effect = StreamParseException(message=words(1000)) + + with TwitterClient([timeline]) as client: + for data, stream in client: + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(len(stream.rule.error), 1024) + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() diff --git a/src/newsreader/news/collection/tests/twitter/collector/__init__.py b/src/newsreader/news/collection/tests/twitter/collector/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/collector/mocks.py b/src/newsreader/news/collection/tests/twitter/collector/mocks.py new file mode 100644 index 0000000..c57f9cf --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/collector/mocks.py @@ -0,0 +1,227 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +empty_mock = [] diff --git a/src/newsreader/news/collection/tests/twitter/collector/tests.py b/src/newsreader/news/collection/tests/twitter/collector/tests.py new file mode 100644 index 0000000..766e971 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/collector/tests.py @@ -0,0 +1,180 @@ +from datetime import datetime +from unittest.mock import patch +from uuid import uuid4 + +from django.test import TestCase +from django.utils import timezone + +import pytz + +from freezegun import freeze_time +from ftfy import fix_text + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamForbiddenException, + StreamNotFoundException, + StreamTimeOutException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.collector.mocks import ( + empty_mock, + simple_mock, +) +from newsreader.news.collection.twitter import TWITTER_URL, TwitterCollector +from newsreader.news.collection.utils import truncate_text +from newsreader.news.core.models import Post + + +@freeze_time("2020-09-26 14:40:00") +class TwitterCollectorTestCase(TestCase): + def setUp(self): + patched_get = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = patched_get.start() + + patched_parse = patch("newsreader.news.collection.twitter.TwitterStream.parse") + self.mocked_parse = patched_parse.start() + + def tearDown(self): + patch.stopall() + + def test_simple_batch(self): + self.mocked_parse.return_value = simple_mock + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + screen_name="RobertsSpaceInd", + enabled=True, + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertCountEqual( + Post.objects.values_list("remote_identifier", flat=True), + ("1307054882210435074", "1307029168941461504"), + ) + + self.assertEquals(timeline.succeeded, True) + self.assertEquals(timeline.last_run, timezone.now()) + self.assertIsNone(timeline.error) + + post = Post.objects.get( + remote_identifier="1307054882210435074", + rule__type=RuleTypeChoices.twitter_timeline, + ) + + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 20, 32, 22)) + ) + + title = truncate_text( + Post, + "title", + "It's a close match-up for #SCShipShowdown today! Which Aegis ship" + " do you think will make it to the Semi-Finals?", + ) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals(post.title, title) + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1307054882210435074" + ) + + post = Post.objects.get( + remote_identifier="1307029168941461504", + rule__type=RuleTypeChoices.twitter_timeline, + ) + + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 18, 50, 11)) + ) + + body = fix_text( + "We\u2019re welcoming members of our Builds, Publishes and Platform" + " teams on Star Citizen Live to talk about the process involved in" + " bringing everyone\u2019s work together and getting it out into your" + " hands. Going live on #Twitch in 10 minutes." + " \ud83c\udfa5\ud83d\udd34 \n\nTune in:" + " https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9" + ) + + title = truncate_text(Post, "title", body) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals(post.title, title) + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1307029168941461504" + ) + + def test_empty_batch(self): + self.mocked_parse.return_value = empty_mock + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + + self.assertEquals(timeline.succeeded, True) + self.assertEquals(timeline.last_run, timezone.now()) + self.assertIsNone(timeline.error) + + def test_not_found(self): + self.mocked_fetch.side_effect = StreamNotFoundException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream not found") + + def test_denied(self): + self.mocked_fetch.side_effect = StreamDeniedException + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream does not have sufficient permissions") + + user = timeline.user + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) + + def test_forbidden(self): + self.mocked_fetch.side_effect = StreamForbiddenException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream forbidden") + + def test_timed_out(self): + self.mocked_fetch.side_effect = StreamTimeOutException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream timed out") diff --git a/src/newsreader/news/collection/tests/twitter/stream/__init__.py b/src/newsreader/news/collection/tests/twitter/stream/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/stream/mocks.py b/src/newsreader/news/collection/tests/twitter/stream/mocks.py new file mode 100644 index 0000000..1b7c6a2 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/stream/mocks.py @@ -0,0 +1,225 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] diff --git a/src/newsreader/news/collection/tests/twitter/stream/tests.py b/src/newsreader/news/collection/tests/twitter/stream/tests.py new file mode 100644 index 0000000..4edb639 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/stream/tests.py @@ -0,0 +1,107 @@ +from json import JSONDecodeError +from unittest.mock import patch + +from django.test import TestCase + +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamForbiddenException, + StreamNotFoundException, + StreamParseException, + StreamTimeOutException, + StreamTooManyException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.stream.mocks import simple_mock +from newsreader.news.collection.twitter import TwitterStream + + +class TwitterStreamTestCase(TestCase): + def setUp(self): + self.patched_fetch = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = self.patched_fetch.start() + + def tearDown(self): + patch.stopall() + + def test_simple_stream(self): + self.mocked_fetch.return_value.json.return_value = simple_mock + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + data, stream = stream.read() + + self.assertEquals(data, simple_mock) + self.assertEquals(stream, stream) + + self.mocked_fetch.assert_called() + + def test_stream_raises_exception(self): + self.mocked_fetch.side_effect = StreamException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_denied_exception(self): + self.mocked_fetch.side_effect = StreamDeniedException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamDeniedException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_not_found_exception(self): + self.mocked_fetch.side_effect = StreamNotFoundException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamNotFoundException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_time_out_exception(self): + self.mocked_fetch.side_effect = StreamTimeOutException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamTimeOutException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_forbidden_exception(self): + self.mocked_fetch.side_effect = StreamForbiddenException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamForbiddenException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_parse_exception(self): + self.mocked_fetch.return_value.json.side_effect = JSONDecodeError( + "No json found", "{}", 5 + ) + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamParseException): + stream.read() + + self.mocked_fetch.assert_called() diff --git a/src/newsreader/news/collection/tests/twitter/test_scheduler.py b/src/newsreader/news/collection/tests/twitter/test_scheduler.py new file mode 100644 index 0000000..a3c2db8 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/test_scheduler.py @@ -0,0 +1,63 @@ +from json import JSONDecodeError +from unittest.mock import patch + +from django.test import TestCase + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.news.collection.exceptions import StreamException +from newsreader.news.collection.twitter import TwitterTimeLineScheduler + + +class TwitterTimeLineSchedulerTestCase(TestCase): + def setUp(self): + patched_fetch = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = patched_fetch.start() + + def test_simple(self): + user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar") + + self.mocked_fetch.return_value.json.return_value = { + "rate_limit_context": {"application": "dummykey"}, + "resources": { + "statuses": { + "/statuses/user_timeline": { + "limit": 1500, + "remaining": 1500, + "reset": 1601141386, + } + } + }, + } + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), 1500) + + def test_stream_exception(self): + user = UserFactory(twitter_oauth_token=None, twitter_oauth_token_secret=None) + + self.mocked_fetch.side_effect = StreamException + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), None) + + def test_json_decode_error(self): + user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar") + + self.mocked_fetch.return_value.json.side_effect = JSONDecodeError( + "foo", "bar", 10 + ) + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), None) + + def test_unexpected_contents(self): + user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar") + + self.mocked_fetch.return_value.json.return_value = {"foo": "bar"} + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), None) diff --git a/src/newsreader/news/collection/tests/utils/tests.py b/src/newsreader/news/collection/tests/utils/tests.py index 10013c3..e88d1bf 100644 --- a/src/newsreader/news/collection/tests/utils/tests.py +++ b/src/newsreader/news/collection/tests/utils/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase @@ -19,7 +19,7 @@ from newsreader.news.collection.utils import fetch, post class HelperFunctionTestCase: def test_simple(self): - self.mocked_method.return_value = MagicMock(status_code=200, content="content") + self.mocked_method.return_value = Mock(status_code=200, content="content") url = "https://www.bbc.co.uk/news" response = self.method(url) @@ -27,7 +27,7 @@ class HelperFunctionTestCase: self.assertEquals(response.content, "content") def test_raises_not_found(self): - self.mocked_method.return_value = MagicMock(status_code=404) + self.mocked_method.return_value = Mock(status_code=404) url = "https://www.bbc.co.uk/news" @@ -35,7 +35,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_denied(self): - self.mocked_method.return_value = MagicMock(status_code=401) + self.mocked_method.return_value = Mock(status_code=401) url = "https://www.bbc.co.uk/news" @@ -43,7 +43,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_forbidden(self): - self.mocked_method.return_value = MagicMock(status_code=403) + self.mocked_method.return_value = Mock(status_code=403) url = "https://www.bbc.co.uk/news" @@ -51,7 +51,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_timed_out(self): - self.mocked_method.return_value = MagicMock(status_code=408) + self.mocked_method.return_value = Mock(status_code=408) url = "https://www.bbc.co.uk/news" @@ -99,7 +99,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_stream_error_on_too_many_requests(self): - self.mocked_method.return_value = MagicMock(status_code=429) + self.mocked_method.return_value = Mock(status_code=429) url = "https://www.bbc.co.uk/news" diff --git a/src/newsreader/news/collection/tests/views/base.py b/src/newsreader/news/collection/tests/views/base.py index d7de171..17f232c 100644 --- a/src/newsreader/news/collection/tests/views/base.py +++ b/src/newsreader/news/collection/tests/views/base.py @@ -49,7 +49,7 @@ class CollectionRuleViewTestCase: timezone=other_rule.timezone, ) - other_url = reverse("news:collection:rule-update", args=[other_rule.pk]) + other_url = reverse("news:collection:feed-update", args=[other_rule.pk]) response = self.client.post(other_url, self.form_data) self.assertEquals(response.status_code, 404) diff --git a/src/newsreader/news/collection/tests/views/test_crud.py b/src/newsreader/news/collection/tests/views/test_crud.py index 61f6835..7da241d 100644 --- a/src/newsreader/news/collection/tests/views/test_crud.py +++ b/src/newsreader/news/collection/tests/views/test_crud.py @@ -3,6 +3,8 @@ from django.urls import reverse import pytz +from django_celery_beat.models import PeriodicTask + from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.tests.factories import FeedFactory @@ -10,11 +12,11 @@ from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCa from newsreader.news.core.tests.factories import CategoryFactory -class CollectionRuleCreateViewTestCase(CollectionRuleViewTestCase, TestCase): +class FeedCreateViewTestCase(CollectionRuleViewTestCase, TestCase): def setUp(self): super().setUp() - self.url = reverse("news:collection:rule-create") + self.url = reverse("news:collection:feed-create") self.form_data.update( name="new rule", @@ -37,15 +39,21 @@ class CollectionRuleCreateViewTestCase(CollectionRuleViewTestCase, TestCase): self.assertEquals(rule.category.pk, self.category.pk) self.assertEquals(rule.user.pk, self.user.pk) + self.assertTrue( + PeriodicTask.objects.get( + name=f"{self.user.email}-feed", task="FeedTask", enabled=True + ) + ) -class CollectionRuleUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): + +class FeedUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): def setUp(self): super().setUp() self.rule = FeedFactory( name="collection rule", user=self.user, category=self.category ) - self.url = reverse("news:collection:rule-update", kwargs={"pk": self.rule.pk}) + self.url = reverse("news:collection:feed-update", kwargs={"pk": self.rule.pk}) self.form_data.update( name=self.rule.name, @@ -94,7 +102,7 @@ class CollectionRuleUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): category=self.category, type=RuleTypeChoices.subreddit, ) - url = reverse("news:collection:rule-update", kwargs={"pk": rule.pk}) + url = reverse("news:collection:feed-update", kwargs={"pk": rule.pk}) response = self.client.get(url) diff --git a/src/newsreader/news/collection/tests/views/test_import_view.py b/src/newsreader/news/collection/tests/views/test_import_view.py index f4188e7..a1f0017 100644 --- a/src/newsreader/news/collection/tests/views/test_import_view.py +++ b/src/newsreader/news/collection/tests/views/test_import_view.py @@ -84,7 +84,7 @@ class OPMLImportTestCase(TestCase): rules = CollectionRule.objects.all() self.assertEquals(len(rules), 0) - self.assertFormError(response, "form", "file", _("No (new) rules found")) + self.assertFormError(response, "form", "file", _("No (new) feeds found")) def test_invalid_feeds(self): file_path = self._get_file_path("invalid-url-feeds.opml") @@ -99,7 +99,7 @@ class OPMLImportTestCase(TestCase): rules = CollectionRule.objects.all() self.assertEquals(len(rules), 0) - self.assertFormError(response, "form", "file", _("No (new) rules found")) + self.assertFormError(response, "form", "file", _("No (new) feeds found")) def test_invalid_file(self): file_path = self._get_file_path("test.png") diff --git a/src/newsreader/news/collection/tests/views/test_twitter_views.py b/src/newsreader/news/collection/tests/views/test_twitter_views.py new file mode 100644 index 0000000..d9afa26 --- /dev/null +++ b/src/newsreader/news/collection/tests/views/test_twitter_views.py @@ -0,0 +1,129 @@ +from django.test import TestCase +from django.urls import reverse + +import pytz + +from django_celery_beat.models import PeriodicTask + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCase +from newsreader.news.collection.twitter import TWITTER_API_URL +from newsreader.news.core.tests.factories import CategoryFactory + + +class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): + def setUp(self): + super().setUp() + + self.form_data = { + "name": "new rule", + "screen_name": "RobertsSpaceInd", + "category": str(self.category.pk), + } + + self.url = reverse("news:collection:twitter-timeline-create") + + def test_creation(self): + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 302) + + rule = CollectionRule.objects.get(name="new rule") + + self.assertEquals(rule.type, RuleTypeChoices.twitter_timeline) + self.assertEquals( + rule.url, + f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended", + ) + self.assertEquals(rule.timezone, str(pytz.utc)) + self.assertEquals(rule.favicon, None) + self.assertEquals(rule.category.pk, self.category.pk) + self.assertEquals(rule.user.pk, self.user.pk) + + self.assertTrue( + PeriodicTask.objects.get( + name=f"{self.user.email}-timeline", + task="TwitterTimelineTask", + enabled=True, + ) + ) + + +class TwitterTimelineUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): + def setUp(self): + super().setUp() + + self.rule = TwitterTimelineFactory( + name="Star citizen", + screen_name="RobertsSpaceInd", + user=self.user, + category=self.category, + type=RuleTypeChoices.twitter_timeline, + ) + self.url = reverse( + "news:collection:twitter-timeline-update", kwargs={"pk": self.rule.pk} + ) + + self.form_data = { + "name": self.rule.name, + "screen_name": self.rule.screen_name, + "category": str(self.category.pk), + "timezone": pytz.utc, + } + + def test_name_change(self): + self.form_data.update(name="Star citizen Twitter") + + response = self.client.post(self.url, self.form_data) + self.assertEquals(response.status_code, 302) + + self.rule.refresh_from_db() + + self.assertEquals(self.rule.name, "Star citizen Twitter") + + def test_category_change(self): + new_category = CategoryFactory(user=self.user) + + self.form_data.update(category=new_category.pk) + + response = self.client.post(self.url, self.form_data) + self.assertEquals(response.status_code, 302) + + self.rule.refresh_from_db() + + self.assertEquals(self.rule.category.pk, new_category.pk) + + def test_twitter_timelines_only(self): + rule = TwitterTimelineFactory( + name="Fake twitter", + user=self.user, + category=self.category, + type=RuleTypeChoices.feed, + url="https://twitter.com/RobertsSpaceInd", + ) + url = reverse("news:collection:twitter-timeline-update", kwargs={"pk": rule.pk}) + + response = self.client.get(url) + + self.assertEquals(response.status_code, 404) + + def test_screen_name_change(self): + self.form_data.update(screen_name="CyberpunkGame") + + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 302) + + self.rule.refresh_from_db() + + self.assertEquals(self.rule.type, RuleTypeChoices.twitter_timeline) + self.assertEquals( + self.rule.url, + f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name=CyberpunkGame&tweet_mode=extended", + ) + self.assertEquals(self.rule.timezone, str(pytz.utc)) + self.assertEquals(self.rule.favicon, None) + self.assertEquals(self.rule.category.pk, self.category.pk) + self.assertEquals(self.rule.user.pk, self.user.pk) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py new file mode 100644 index 0000000..dc32ecc --- /dev/null +++ b/src/newsreader/news/collection/twitter.py @@ -0,0 +1,281 @@ +import logging + +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from json import JSONDecodeError + +from django.conf import settings +from django.utils import timezone +from django.utils.html import format_html, urlize + +import pytz + +from ftfy import fix_text +from requests_oauthlib import OAuth1 as OAuth + +from newsreader.news.collection.base import ( + PostBuilder, + PostClient, + PostCollector, + PostStream, + Scheduler, +) +from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamNotFoundException, + StreamParseException, + StreamTimeOutException, + StreamTooManyException, +) +from newsreader.news.collection.utils import fetch, truncate_text +from newsreader.news.core.models import Post + + +logger = logging.getLogger(__name__) + +TWITTER_URL = "https://twitter.com" +TWITTER_API_URL = "https://api.twitter.com/1.1" +TWITTER_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" +TWITTER_AUTH_URL = "https://api.twitter.com/oauth/authorize" +TWITTER_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" +TWITTER_REVOKE_URL = f"{TWITTER_API_URL}/oauth/invalidate_token" + + +class TwitterBuilder(PostBuilder): + rule_type = RuleTypeChoices.twitter_timeline + + def build(self): + results = {} + rule = self.stream.rule + + for post in self.payload: + remote_identifier = post["id_str"] + + if remote_identifier in self.existing_posts: + continue + + url = f"{TWITTER_URL}/{rule.screen_name}/status/{remote_identifier}" + body = urlize(post["full_text"], nofollow=True) + title = truncate_text( + Post, "title", self.sanitize_fragment(post["full_text"]) + ) + + publication_date = pytz.utc.localize( + datetime.strptime(post["created_at"], "%a %b %d %H:%M:%S +0000 %Y") + ) + + if "extended_entities" in post: + try: + media_entities = self.get_media_entities(post) + body += media_entities + except KeyError: + logger.exception(f"Failed parsing media_entities for {url}") + + if "retweeted_status" in post: + original_post = post["retweeted_status"] + original_tweet = urlize(original_post["full_text"], nofollow=True) + body = f"{body}
    Original tweet: {original_tweet}
    " + if "quoted_status" in post: + original_post = post["quoted_status"] + original_tweet = urlize(original_post["full_text"], nofollow=True) + body = f"{body}
    Quoted tweet: {original_tweet}
    " + + body = self.sanitize_fragment(body) + + data = { + "remote_identifier": remote_identifier, + "title": fix_text(title), + "body": fix_text(body), + "author": rule.screen_name, + "publication_date": publication_date, + "url": url, + "rule": rule, + } + + results[remote_identifier] = Post(**data) + + self.instances = results.values() + + def get_media_entities(self, post): + media_entities = post["extended_entities"]["media"] + formatted_entities = "" + + for media_entity in media_entities: + media_type = media_entity["type"] + media_url = media_entity["media_url_https"] + title = media_entity["id_str"] + + if media_type == TwitterPostTypeChoices.photo: + html_fragment = format_html( + """
    {title}
    """, + title=title, + media_url=media_url, + ) + + formatted_entities += html_fragment + + elif media_type in ( + TwitterPostTypeChoices.video, + TwitterPostTypeChoices.animated_gif, + ): + meta_data = media_entity["video_info"] + + videos = sorted( + [video for video in meta_data["variants"]], + reverse=True, + key=lambda video: video.get("bitrate", 0), + ) + + if not videos: + continue + + video = videos[0] + content_type = video["content_type"] + url = video["url"] + + html_fragment = format_html( + """
    """, + url=url, + content_type=content_type, + ) + + formatted_entities += html_fragment + + return formatted_entities + + +class TwitterStream(PostStream): + rule_type = RuleTypeChoices.twitter_timeline + + def read(self): + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=self.rule.user.twitter_oauth_token, + resource_owner_secret=self.rule.user.twitter_oauth_token_secret, + ) + + response = fetch(self.rule.url, auth=oauth) + + return self.parse(response), self + + def parse(self, response): + try: + return response.json() + except JSONDecodeError as e: + raise StreamParseException( + response=response, message="Failed parsing json" + ) from e + + +class TwitterClient(PostClient): + stream = TwitterStream + + def __enter__(self): + streams = [self.stream(timeline) for timeline in self.rules] + + with ThreadPoolExecutor(max_workers=10) as executor: + futures = {executor.submit(stream.read): stream for stream in streams} + + for future in as_completed(futures): + stream = futures[future] + + try: + payload = future.result() + + stream.rule.error = None + stream.rule.succeeded = True + + yield payload + except StreamTooManyException as e: + logger.exception("Ratelimit hit, aborting twitter calls") + + self.set_rule_error(stream.rule, e) + + break + except StreamDeniedException as e: + logger.warning( + f"Access token expired for user {stream.rule.user.pk}" + ) + + stream.rule.user.twitter_oauth_token = None + stream.rule.user.twitter_oauth_token_secret = None + stream.rule.user.save() + + self.set_rule_error(stream.rule, e) + + break + except (StreamNotFoundException, StreamTimeOutException) as e: + logger.warning(f"Request failed for {stream.rule.screen_name}") + + self.set_rule_error(stream.rule, e) + + continue + except StreamException as e: + logger.exception(f"Request failed for {stream.rule.screen_name}") + + self.set_rule_error(stream.rule, e) + + continue + finally: + stream.rule.last_run = timezone.now() + stream.rule.save() + + +class TwitterCollector(PostCollector): + builder = TwitterBuilder + client = TwitterClient + + +# see https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits +class TwitterTimeLineScheduler(Scheduler): + def __init__(self, user, timelines=[]): + self.user = user + + if not timelines: + self.timelines = ( + user.rules.enabled() + .filter(type=RuleTypeChoices.twitter_timeline) + .order_by("last_run")[:200] + ) + else: + self.timelines = timelines + + def get_scheduled_rules(self): + max_amount = self.get_current_ratelimit() + return self.timelines[:max_amount] if max_amount else [] + + def get_current_ratelimit(self): + endpoint = "application/rate_limit_status.json?resources=statuses" + + if ( + not self.user.twitter_oauth_token + or not self.user.twitter_oauth_token_secret + ): + return + + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=self.user.twitter_oauth_token, + resource_owner_secret=self.user.twitter_oauth_token_secret, + ) + + try: + response = fetch(f"{TWITTER_API_URL}/{endpoint}", auth=oauth) + except StreamException: + logger.exception(f"Unable to retrieve current ratelimit for {self.user.pk}") + return + + try: + payload = response.json() + except JSONDecodeError: + logger.exception(f"Unable to parse ratelimit request for {self.user.pk}") + return + + try: + return payload["resources"]["statuses"]["/statuses/user_timeline"]["limit"] + except KeyError: + return diff --git a/src/newsreader/news/collection/urls.py b/src/newsreader/news/collection/urls.py index 5253210..7d883f2 100644 --- a/src/newsreader/news/collection/urls.py +++ b/src/newsreader/news/collection/urls.py @@ -11,12 +11,14 @@ from newsreader.news.collection.views import ( CollectionRuleBulkDeleteView, CollectionRuleBulkDisableView, CollectionRuleBulkEnableView, - CollectionRuleCreateView, CollectionRuleListView, - CollectionRuleUpdateView, + FeedCreateView, + FeedUpdateView, OPMLImportView, SubRedditCreateView, SubRedditUpdateView, + TwitterTimelineCreateView, + TwitterTimelineUpdateView, ) @@ -28,17 +30,13 @@ endpoints = [ ] urlpatterns = [ + # Feeds + path( + "feeds//", login_required(FeedUpdateView.as_view()), name="feed-update" + ), + path("feeds/create/", login_required(FeedCreateView.as_view()), name="feed-create"), + # Generic rules path("rules/", login_required(CollectionRuleListView.as_view()), name="rules"), - path( - "rules//", - login_required(CollectionRuleUpdateView.as_view()), - name="rule-update", - ), - path( - "rules/create/", - login_required(CollectionRuleCreateView.as_view()), - name="rule-create", - ), path( "rules/delete/", login_required(CollectionRuleBulkDeleteView.as_view()), @@ -54,15 +52,27 @@ urlpatterns = [ login_required(CollectionRuleBulkDisableView.as_view()), name="rules-disable", ), + path("rules/import/", login_required(OPMLImportView.as_view()), name="import"), + # Reddit path( - "rules/subreddits/create/", + "subreddits/create/", login_required(SubRedditCreateView.as_view()), name="subreddit-create", ), path( - "rules/subreddits//", + "subreddits//", login_required(SubRedditUpdateView.as_view()), name="subreddit-update", ), - path("rules/import/", login_required(OPMLImportView.as_view()), name="import"), + # Twitter + path( + "twitter/timelines/create/", + login_required(TwitterTimelineCreateView.as_view()), + name="twitter-timeline-create", + ), + path( + "twitter/timelines//", + login_required(TwitterTimelineUpdateView.as_view()), + name="twitter-timeline-update", + ), ] diff --git a/src/newsreader/news/collection/utils.py b/src/newsreader/news/collection/utils.py index 4cfc0e7..0eb1dc0 100644 --- a/src/newsreader/news/collection/utils.py +++ b/src/newsreader/news/collection/utils.py @@ -25,12 +25,12 @@ def build_publication_date(dt, tz): return published_parsed.astimezone(pytz.utc) -def fetch(url, headers={}): +def fetch(url, auth=None, headers={}): headers = {**DEFAULT_HEADERS, **headers} with ResponseHandler() as response_handler: try: - response = requests.get(url, headers=headers) + response = requests.get(url, auth=auth, headers=headers) response_handler.handle_response(response) except RequestException as exception: response_handler.map_exception(exception) diff --git a/src/newsreader/news/collection/views/__init__.py b/src/newsreader/news/collection/views/__init__.py index 20769f3..c66c5a5 100644 --- a/src/newsreader/news/collection/views/__init__.py +++ b/src/newsreader/news/collection/views/__init__.py @@ -1,3 +1,8 @@ +from newsreader.news.collection.views.feed import ( + FeedCreateView, + FeedUpdateView, + OPMLImportView, +) from newsreader.news.collection.views.reddit import ( SubRedditCreateView, SubRedditUpdateView, @@ -6,8 +11,9 @@ from newsreader.news.collection.views.rules import ( CollectionRuleBulkDeleteView, CollectionRuleBulkDisableView, CollectionRuleBulkEnableView, - CollectionRuleCreateView, CollectionRuleListView, - CollectionRuleUpdateView, - OPMLImportView, +) +from newsreader.news.collection.views.twitter import ( + TwitterTimelineCreateView, + TwitterTimelineUpdateView, ) diff --git a/src/newsreader/news/collection/views/base.py b/src/newsreader/news/collection/views/base.py index e7f7b63..d7a3a4d 100644 --- a/src/newsreader/news/collection/views/base.py +++ b/src/newsreader/news/collection/views/base.py @@ -1,8 +1,11 @@ +import json + from django.urls import reverse_lazy import pytz -from newsreader.news.collection.forms import CollectionRuleForm +from django_celery_beat.models import IntervalSchedule, PeriodicTask + from newsreader.news.collection.models import CollectionRule from newsreader.news.core.models import Category @@ -17,7 +20,6 @@ class CollectionRuleViewMixin: class CollectionRuleDetailMixin: success_url = reverse_lazy("news:collection:rules") - form_class = CollectionRuleForm def get_context_data(self, **kwargs): context_data = super().get_context_data(**kwargs) @@ -34,3 +36,25 @@ class CollectionRuleDetailMixin: kwargs = super().get_form_kwargs() kwargs["user"] = self.request.user return kwargs + + +class TaskCreationMixin: + def form_valid(self, form): + response = super().form_valid(form) + + interval, period = self.task_interval + task_interval, _ = IntervalSchedule.objects.get_or_create( + every=interval, period=period + ) + + PeriodicTask.objects.get_or_create( + name=f"{self.request.user.email}-{self.task_name}", + task=self.task_type, + defaults={ + "args": json.dumps([self.request.user.pk]), + "interval": task_interval, + "enabled": True, + }, + ) + + return response diff --git a/src/newsreader/news/collection/views/feed.py b/src/newsreader/news/collection/views/feed.py new file mode 100644 index 0000000..b7803d2 --- /dev/null +++ b/src/newsreader/news/collection/views/feed.py @@ -0,0 +1,70 @@ +from django.contrib import messages +from django.urls import reverse +from django.utils.translation import gettext as _ +from django.views.generic.edit import CreateView, FormView, UpdateView + +from django_celery_beat.models import IntervalSchedule + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms import ( + CollectionRuleBulkForm, + FeedForm, + OPMLImportForm, +) +from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.views.base import ( + CollectionRuleDetailMixin, + CollectionRuleViewMixin, + TaskCreationMixin, +) +from newsreader.utils.opml import parse_opml + + +class FeedUpdateView(CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView): + template_name = "news/collection/views/feed-update.html" + context_object_name = "feed" + form_class = FeedForm + + def get_queryset(self): + queryset = super().get_queryset() + return queryset.filter(type=RuleTypeChoices.feed) + + +class FeedCreateView( + CollectionRuleViewMixin, CollectionRuleDetailMixin, TaskCreationMixin, CreateView +): + template_name = "news/collection/views/feed-create.html" + task_interval = (1, IntervalSchedule.HOURS) + task_name = "feed" + task_type = "FeedTask" + form_class = FeedForm + + +class OPMLImportView(FormView): + form_class = OPMLImportForm + template_name = "news/collection/views/import.html" + + def form_valid(self, form): + user = self.request.user + file = form.cleaned_data["file"] + skip_existing = form.cleaned_data["skip_existing"] + + instances = parse_opml(file, user, skip_existing=skip_existing) + + try: + feeds = CollectionRule.objects.bulk_create(instances) + except IOError: + form.add_error("file", _("Invalid OPML file")) + return self.form_invalid(form) + + if not feeds: + form.add_error("file", _("No (new) feeds found")) + return self.form_invalid(form) + + message = _(f"{len(feeds)} new feeds created") + messages.success(self.request, message) + + return super().form_valid(form) + + def get_success_url(self): + return reverse("news:collection:rules") diff --git a/src/newsreader/news/collection/views/reddit.py b/src/newsreader/news/collection/views/reddit.py index 62ec408..4e44e3f 100644 --- a/src/newsreader/news/collection/views/reddit.py +++ b/src/newsreader/news/collection/views/reddit.py @@ -1,7 +1,7 @@ from django.views.generic.edit import CreateView, UpdateView from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.forms import SubRedditRuleForm +from newsreader.news.collection.forms import SubRedditForm from newsreader.news.collection.views.base import ( CollectionRuleDetailMixin, CollectionRuleViewMixin, @@ -11,14 +11,14 @@ from newsreader.news.collection.views.base import ( class SubRedditCreateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView ): - form_class = SubRedditRuleForm + form_class = SubRedditForm template_name = "news/collection/views/subreddit-create.html" class SubRedditUpdateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView ): - form_class = SubRedditRuleForm + form_class = SubRedditForm template_name = "news/collection/views/subreddit-update.html" context_object_name = "subreddit" diff --git a/src/newsreader/news/collection/views/rules.py b/src/newsreader/news/collection/views/rules.py index e020b67..902eedf 100644 --- a/src/newsreader/news/collection/views/rules.py +++ b/src/newsreader/news/collection/views/rules.py @@ -2,17 +2,14 @@ from django.contrib import messages from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext as _ -from django.views.generic.edit import CreateView, FormView, UpdateView +from django.views.generic.edit import FormView from django.views.generic.list import ListView -from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.forms import CollectionRuleBulkForm, OPMLImportForm -from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.forms import CollectionRuleBulkForm from newsreader.news.collection.views.base import ( CollectionRuleDetailMixin, CollectionRuleViewMixin, ) -from newsreader.utils.opml import parse_opml class CollectionRuleListView(CollectionRuleViewMixin, ListView): @@ -21,23 +18,6 @@ class CollectionRuleListView(CollectionRuleViewMixin, ListView): context_object_name = "rules" -class CollectionRuleUpdateView( - CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView -): - template_name = "news/collection/views/rule-update.html" - context_object_name = "rule" - - def get_queryset(self): - queryset = super().get_queryset() - return queryset.filter(type=RuleTypeChoices.feed) - - -class CollectionRuleCreateView( - CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView -): - template_name = "news/collection/views/rule-create.html" - - class CollectionRuleBulkView(FormView): form_class = CollectionRuleBulkForm @@ -90,33 +70,3 @@ class CollectionRuleBulkDeleteView(CollectionRuleBulkView): rule.delete() return response - - -class OPMLImportView(FormView): - form_class = OPMLImportForm - template_name = "news/collection/views/import.html" - - def form_valid(self, form): - user = self.request.user - file = form.cleaned_data["file"] - skip_existing = form.cleaned_data["skip_existing"] - - instances = parse_opml(file, user, skip_existing=skip_existing) - - try: - rules = CollectionRule.objects.bulk_create(instances) - except IOError: - form.add_error("file", _("Invalid OPML file")) - return self.form_invalid(form) - - if not rules: - form.add_error("file", _("No (new) rules found")) - return self.form_invalid(form) - - message = _(f"{len(rules)} new rules created") - messages.success(self.request, message) - - return super().form_valid(form) - - def get_success_url(self): - return reverse("news:collection:rules") diff --git a/src/newsreader/news/collection/views/twitter.py b/src/newsreader/news/collection/views/twitter.py new file mode 100644 index 0000000..0221a75 --- /dev/null +++ b/src/newsreader/news/collection/views/twitter.py @@ -0,0 +1,33 @@ +from django.views.generic.edit import CreateView, UpdateView + +from django_celery_beat.models import IntervalSchedule + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms import TwitterTimelineForm +from newsreader.news.collection.views.base import ( + CollectionRuleDetailMixin, + CollectionRuleViewMixin, + TaskCreationMixin, +) + + +class TwitterTimelineCreateView( + CollectionRuleViewMixin, CollectionRuleDetailMixin, TaskCreationMixin, CreateView +): + form_class = TwitterTimelineForm + template_name = "news/collection/views/twitter/timeline-create.html" + task_interval = (10, IntervalSchedule.MINUTES) + task_name = "timeline" + task_type = "TwitterTimelineTask" + + +class TwitterTimelineUpdateView( + CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView +): + form_class = TwitterTimelineForm + template_name = "news/collection/views/twitter/timeline-update.html" + context_object_name = "timeline" + + def get_queryset(self): + queryset = super().get_queryset() + return queryset.filter(type=RuleTypeChoices.twitter_timeline) diff --git a/src/newsreader/news/core/templates/news/core/views/categories.html b/src/newsreader/news/core/templates/news/core/views/categories.html index 35fc741..6a6cdae 100644 --- a/src/newsreader/news/core/templates/news/core/views/categories.html +++ b/src/newsreader/news/core/templates/news/core/views/categories.html @@ -30,5 +30,8 @@ ] + {{ categories_update_url|json_script:"updateUrl" }} + {{ categories_create_url|json_script:"createUrl" }} + {{ block.super }} {% endblock %} diff --git a/src/newsreader/news/core/templates/news/core/views/homepage.html b/src/newsreader/news/core/templates/news/core/views/homepage.html index 79e1ccc..502ef63 100644 --- a/src/newsreader/news/core/templates/news/core/views/homepage.html +++ b/src/newsreader/news/core/templates/news/core/views/homepage.html @@ -3,4 +3,13 @@ {% block content %}
    -{% endblock %} +{% endblock content %} + +{% block scripts %} + {{ feed_url|json_script:"feedUrl" }} + {{ subreddit_url|json_script:"subredditUrl" }} + {{ twitter_timeline_url|json_script:"timelineUrl" }} + {{ categories_url|json_script:"categoriesUrl" }} + + {{ block.super }} +{% endblock scripts %} diff --git a/src/newsreader/news/core/views.py b/src/newsreader/news/core/views.py index 9ef81eb..981e7b2 100644 --- a/src/newsreader/news/core/views.py +++ b/src/newsreader/news/core/views.py @@ -11,24 +11,21 @@ from newsreader.news.core.models import Category class NewsView(TemplateView): template_name = "news/core/views/homepage.html" - # TODO serialize objects to show filled main page def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - user = self.request.user - categories = { - category: category.rules.order_by("-created") - for category in user.categories.order_by("name") + return { + **context, + "feed_url": reverse_lazy("news:collection:feed-update", args=(0,)), + "subreddit_url": reverse_lazy( + "news:collection:subreddit-update", args=(0,) + ), + "twitter_timeline_url": reverse_lazy( + "news:collection:twitter-timeline-update", args=(0,) + ), + "categories_url": reverse_lazy("news:core:category-update", args=(0,)), } - rules = { - rule: rule.posts.order_by("-publication_date")[:30] - for rule in user.rules.order_by("-created") - } - - context.update(categories=categories, rules=rules) - return context - class CategoryViewMixin: queryset = Category.objects.prefetch_related("rules").order_by("name") @@ -58,6 +55,17 @@ class CategoryListView(CategoryViewMixin, ListView): template_name = "news/core/views/categories.html" context_object_name = "categories" + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + + return { + **context, + "categories_create_url": reverse_lazy("news:core:category-create"), + "categories_update_url": ( + reverse_lazy("news:core:category-update", args=(0,)) + ), + } + class CategoryUpdateView(CategoryViewMixin, CategoryDetailMixin, UpdateView): template_name = "news/core/views/category-update.html" diff --git a/src/newsreader/scss/components/header/_header.scss b/src/newsreader/scss/components/header/_header.scss new file mode 100644 index 0000000..ed96dc6 --- /dev/null +++ b/src/newsreader/scss/components/header/_header.scss @@ -0,0 +1,3 @@ +.header { + padding: 15px; +} diff --git a/src/newsreader/scss/components/header/index.scss b/src/newsreader/scss/components/header/index.scss new file mode 100644 index 0000000..5c23e3e --- /dev/null +++ b/src/newsreader/scss/components/header/index.scss @@ -0,0 +1 @@ +@import './header'; diff --git a/src/newsreader/scss/components/index.scss b/src/newsreader/scss/components/index.scss index cc9e717..b82a22d 100644 --- a/src/newsreader/scss/components/index.scss +++ b/src/newsreader/scss/components/index.scss @@ -8,6 +8,7 @@ @import './card/index'; @import './list/index'; +@import './header/index'; @import './messages/index'; @import './section/index'; @import './errorlist/index'; @@ -16,6 +17,8 @@ @import './sidebar/index'; @import './table/index'; +@import './integrations/index'; + @import './rules/index'; @import './category/index'; diff --git a/src/newsreader/scss/components/integrations/_integrations.scss b/src/newsreader/scss/components/integrations/_integrations.scss new file mode 100644 index 0000000..815184e --- /dev/null +++ b/src/newsreader/scss/components/integrations/_integrations.scss @@ -0,0 +1,12 @@ +.integrations { + display: flex; + flex-direction: column; + gap: 15px; + + padding: 15px; + + &__controls { + display: flex; + gap: 10px; + } +} diff --git a/src/newsreader/scss/components/integrations/index.scss b/src/newsreader/scss/components/integrations/index.scss new file mode 100644 index 0000000..7f9e759 --- /dev/null +++ b/src/newsreader/scss/components/integrations/index.scss @@ -0,0 +1 @@ +@import './integrations'; diff --git a/src/newsreader/scss/elements/button/_button.scss b/src/newsreader/scss/elements/button/_button.scss index a8eb3bc..7cd062a 100644 --- a/src/newsreader/scss/elements/button/_button.scss +++ b/src/newsreader/scss/elements/button/_button.scss @@ -44,10 +44,24 @@ &--reddit { color: $white !important; - background-color: lighten($reddit-orange, 5%); + background-color: $reddit-orange; &:hover { - background-color: $reddit-orange; + background-color: lighten($reddit-orange, 5%); } } + + &--twitter { + color: $white !important; + background-color: $twitter-blue; + + &:hover { + background-color: lighten($twitter-blue, 5%); + } + } + + &--disabled { + color: $font-color !important; + background-color: $gray !important; + } } diff --git a/src/newsreader/scss/pages/index.scss b/src/newsreader/scss/pages/index.scss index 44ca8a7..2ac0bb2 100644 --- a/src/newsreader/scss/pages/index.scss +++ b/src/newsreader/scss/pages/index.scss @@ -12,3 +12,4 @@ @import './rules/index'; @import './settings/index'; +@import './integrations/index'; diff --git a/src/newsreader/scss/pages/integrations/index.scss b/src/newsreader/scss/pages/integrations/index.scss new file mode 100644 index 0000000..ccf52c3 --- /dev/null +++ b/src/newsreader/scss/pages/integrations/index.scss @@ -0,0 +1,5 @@ +#integrations--page { + .section { + width: 70%; + } +} diff --git a/src/newsreader/scss/partials/_colors.scss b/src/newsreader/scss/partials/_colors.scss index b2f124d..87f6e49 100644 --- a/src/newsreader/scss/partials/_colors.scss +++ b/src/newsreader/scss/partials/_colors.scss @@ -12,6 +12,7 @@ $font-color: rgba(48, 51, 53, 1); $header-color: rgba(100, 101, 102, 1); $reddit-orange: rgba(255, 69, 0, 1); +$twitter-blue: rgba(29, 155, 240, 1); $transparant-red: transparentize($red, 0.8); $transparant-blue: transparentize($blue, 0.8); diff --git a/src/newsreader/templates/components/form/form.html b/src/newsreader/templates/components/form/form.html index e183c25..9f1ab47 100644 --- a/src/newsreader/templates/components/form/form.html +++ b/src/newsreader/templates/components/form/form.html @@ -4,7 +4,7 @@ {% csrf_token %} {% if title %} - {% include "components/form/title.html" with title=title only %} + {% include "components/header/header.html" with title=title only %} {% endif %} {% block intro %} diff --git a/src/newsreader/templates/components/form/title.html b/src/newsreader/templates/components/form/title.html deleted file mode 100644 index 3adcb75..0000000 --- a/src/newsreader/templates/components/form/title.html +++ /dev/null @@ -1,3 +0,0 @@ -
    -

    {{ title }}

    -
    diff --git a/src/newsreader/templates/components/header/header.html b/src/newsreader/templates/components/header/header.html new file mode 100644 index 0000000..c21c233 --- /dev/null +++ b/src/newsreader/templates/components/header/header.html @@ -0,0 +1,3 @@ +
    +

    {{ title }}

    +
    diff --git a/src/newsreader/utils/opml.py b/src/newsreader/utils/opml.py index 55a9387..1aca0fd 100644 --- a/src/newsreader/utils/opml.py +++ b/src/newsreader/utils/opml.py @@ -38,4 +38,5 @@ def parse_opml(file, user, skip_existing=False): logging.info(f"Skipped due to invalid URL: {e}") continue + # TODO create feed type rules yield CollectionRule(url=feed_url, name=name, user=user) diff --git a/webpack.common.babel.js b/webpack.common.babel.js index 4ad1700..bbfb403 100644 --- a/webpack.common.babel.js +++ b/webpack.common.babel.js @@ -26,8 +26,9 @@ export default { use: { loader: 'file-loader', options: { - name: 'fonts/[name].[ext]', - publicPath: '../', + name: '[name].[ext]', + outputPath: 'fonts', + publicPath: '/static/fonts/', }, }, },