diff --git a/gulp/sass.js b/gulp/sass.js index 46cd291..3847e37 100644 --- a/gulp/sass.js +++ b/gulp/sass.js @@ -28,6 +28,7 @@ const taskMappings = [ { name: 'category', destDir: `${CORE_DIR}/${STATIC_SUFFIX}` }, { name: 'rules', destDir: `${COLLECTION_DIR}/${STATIC_SUFFIX}` }, { name: 'rule', destDir: `${COLLECTION_DIR}/${STATIC_SUFFIX}` }, + { name: 'import', destDir: `${COLLECTION_DIR}/${STATIC_SUFFIX}` }, ]; export const sassTask = done => { diff --git a/src/newsreader/conf/base.py b/src/newsreader/conf/base.py index 6f19f91..c0a84c1 100644 --- a/src/newsreader/conf/base.py +++ b/src/newsreader/conf/base.py @@ -12,9 +12,10 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ import os +from pathlib import Path -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ diff --git a/src/newsreader/js/pages/rules/App.js b/src/newsreader/js/pages/rules/App.js index fd01229..7ceae4a 100644 --- a/src/newsreader/js/pages/rules/App.js +++ b/src/newsreader/js/pages/rules/App.js @@ -74,9 +74,15 @@ class App extends React.Component { const pageHeader = ( <>

Rules

- - Create rule - + +
+ + Import rules + + + Create rule + +
); diff --git a/src/newsreader/news/collection/forms.py b/src/newsreader/news/collection/forms.py index 89c72e9..2213840 100644 --- a/src/newsreader/news/collection/forms.py +++ b/src/newsreader/news/collection/forms.py @@ -28,3 +28,8 @@ class CollectionRuleForm(forms.ModelForm): class Meta: model = CollectionRule fields = ("name", "url", "timezone", "favicon", "category") + + +class OPMLImportForm(forms.Form): + file = forms.FileField(allow_empty_file=False) + skip_existing = forms.BooleanField(initial=False, required=False) diff --git a/src/newsreader/news/collection/templates/collection/import.html b/src/newsreader/news/collection/templates/collection/import.html new file mode 100644 index 0000000..28785b8 --- /dev/null +++ b/src/newsreader/news/collection/templates/collection/import.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} + +{% load static i18n %} + +{% block head %} + +{% endblock %} + +{% block content %} +
+
+ {% csrf_token %} + {{ form.non_field_errors }} + +
+

{% trans "Import an OPML file" %}

+
+
+
+ + {{ form.file.errors }} + {{ form.file }} +
+ +
+ + {{ form.skip_existing }} +
+ +
+ Cancel + +
+
+
+
+{% endblock %} diff --git a/src/newsreader/news/collection/tests/test_views.py b/src/newsreader/news/collection/tests/test_views.py index d6a478e..5a1a59e 100644 --- a/src/newsreader/news/collection/tests/test_views.py +++ b/src/newsreader/news/collection/tests/test_views.py @@ -1,5 +1,9 @@ +import os + +from django.conf import settings from django.test import Client, TestCase from django.urls import reverse +from django.utils.translation import gettext_lazy as _ import pytz @@ -146,3 +150,131 @@ class CollectionRuleUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): self.rule.refresh_from_db() self.assertEquals(self.rule.category, None) + + +class OPMLImportTestCase(TestCase): + def setUp(self): + self.user = UserFactory(password="test") + self.client.login(email=self.user.email, password="test") + + self.form_data = {"file": "", "skip_existing": False} + self.url = reverse("import") + + def _get_file_path(self, name): + file_dir = os.path.join(settings.BASE_DIR, "utils", "tests", "files") + return os.path.join(file_dir, name) + + def test_simple(self): + file_path = self._get_file_path("feeds.opml") + + with open(file_path) as file: + self.form_data.update(file=file) + + response = self.client.post(self.url, self.form_data) + + self.assertRedirects(response, reverse("rules")) + + rules = CollectionRule.objects.all() + self.assertEquals(len(rules), 4) + + def test_existing_rules(self): + CollectionRuleFactory(url="http://www.engadget.com/rss-full.xml", user=self.user) + CollectionRuleFactory(url="https://news.ycombinator.com/rss", user=self.user) + CollectionRuleFactory( + url="http://feeds.feedburner.com/Techcrunch", user=self.user + ) + CollectionRuleFactory( + url="http://feeds.feedburner.com/tweakers/nieuws", user=self.user + ) + + file_path = self._get_file_path("feeds.opml") + + with open(file_path) as file: + self.form_data.update(file=file) + + response = self.client.post(self.url, self.form_data) + + self.assertRedirects(response, reverse("rules")) + + rules = CollectionRule.objects.all() + self.assertEquals(len(rules), 8) + + def test_skip_existing_rules(self): + CollectionRuleFactory(url="http://www.engadget.com/rss-full.xml", user=self.user) + CollectionRuleFactory(url="https://news.ycombinator.com/rss", user=self.user) + CollectionRuleFactory( + url="http://feeds.feedburner.com/Techcrunch", user=self.user + ) + CollectionRuleFactory( + url="http://feeds.feedburner.com/tweakers/nieuws", user=self.user + ) + + file_path = self._get_file_path("feeds.opml") + + with open(file_path) as file: + self.form_data.update(file=file, skip_existing=True) + + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 200) + + rules = CollectionRule.objects.all() + self.assertEquals(len(rules), 4) + + def test_empty_feed_file(self): + file_path = self._get_file_path("empty-feeds.opml") + + with open(file_path) as file: + self.form_data.update(file=file) + + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 200) + + rules = CollectionRule.objects.all() + self.assertEquals(len(rules), 0) + + self.assertFormError(response, "form", "file", _("No (new) rules found")) + + def test_invalid_feeds(self): + file_path = self._get_file_path("invalid-url-feeds.opml") + + with open(file_path) as file: + self.form_data.update(file=file) + + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 200) + + rules = CollectionRule.objects.all() + + self.assertEquals(len(rules), 0) + self.assertFormError(response, "form", "file", _("No (new) rules found")) + + def test_invalid_file(self): + file_path = self._get_file_path("test.png") + + with open(file_path, "rb") as file: + self.form_data.update(file=file) + + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 200) + + rules = CollectionRule.objects.all() + self.assertEquals(len(rules), 0) + + self.assertFormError(response, "form", "file", _("Invalid OPML file")) + + def test_feeds_with_missing_attr(self): + file_path = self._get_file_path("missing-feeds.opml") + + with open(file_path) as file: + self.form_data.update(file=file) + + response = self.client.post(self.url, self.form_data) + + self.assertRedirects(response, reverse("rules")) + + rules = CollectionRule.objects.all() + self.assertEquals(len(rules), 2) diff --git a/src/newsreader/news/collection/urls.py b/src/newsreader/news/collection/urls.py index 16c80b7..28b6f38 100644 --- a/src/newsreader/news/collection/urls.py +++ b/src/newsreader/news/collection/urls.py @@ -11,6 +11,7 @@ from newsreader.news.collection.views import ( CollectionRuleCreateView, CollectionRuleListView, CollectionRuleUpdateView, + OPMLImportView, ) @@ -33,4 +34,5 @@ urlpatterns = [ login_required(CollectionRuleCreateView.as_view()), name="rule-create", ), + path("rules/import/", login_required(OPMLImportView.as_view()), name="import"), ] diff --git a/src/newsreader/news/collection/views.py b/src/newsreader/news/collection/views.py index 8919ddf..b62542c 100644 --- a/src/newsreader/news/collection/views.py +++ b/src/newsreader/news/collection/views.py @@ -1,14 +1,17 @@ from typing import Dict, Iterable +from django.contrib import messages from django.urls import reverse_lazy -from django.views.generic.edit import CreateView, UpdateView +from django.utils.translation import gettext_lazy as _ +from django.views.generic.edit import CreateView, FormView, UpdateView from django.views.generic.list import ListView import pytz -from newsreader.news.collection.forms import CollectionRuleForm +from newsreader.news.collection.forms import CollectionRuleForm, OPMLImportForm from newsreader.news.collection.models import CollectionRule from newsreader.news.core.models import Category +from newsreader.utils.opml import parse_opml class CollectionRuleViewMixin: @@ -56,3 +59,31 @@ class CollectionRuleCreateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView ): template_name = "collection/rule-create.html" + + +class OPMLImportView(FormView): + form_class = OPMLImportForm + success_url = reverse_lazy("rules") + template_name = "collection/import.html" + + def form_valid(self, form): + user = self.request.user + file = form.cleaned_data["file"] + skip_existing = form.cleaned_data["skip_existing"] + + instances = parse_opml(file, user, skip_existing=skip_existing) + + try: + rules = CollectionRule.objects.bulk_create(instances) + except IOError: + form.add_error("file", _("Invalid OPML file")) + return self.form_invalid(form) + + if not rules: + form.add_error("file", _("No (new) rules found")) + return self.form_invalid(form) + + message = _(f"{len(rules)} new rules created") + messages.success(self.request, message) + + return super().form_valid(form) diff --git a/src/newsreader/scss/pages/import/components/import-form/_import-form.scss b/src/newsreader/scss/pages/import/components/import-form/_import-form.scss new file mode 100644 index 0000000..19acc5c --- /dev/null +++ b/src/newsreader/scss/pages/import/components/import-form/_import-form.scss @@ -0,0 +1,17 @@ +.import-form { + margin: 20px 0; + + &__fieldset:last-child { + display: flex; + flex-direction: row; + justify-content: space-between; + } + + & input[type=file] { + width: 50%; + } + + & input[type=checkbox] { + margin: 0 auto 0 10px; + } +} diff --git a/src/newsreader/scss/pages/import/components/import-form/index.scss b/src/newsreader/scss/pages/import/components/import-form/index.scss new file mode 100644 index 0000000..b6407f2 --- /dev/null +++ b/src/newsreader/scss/pages/import/components/import-form/index.scss @@ -0,0 +1 @@ +@import "import-form"; diff --git a/src/newsreader/scss/pages/import/components/index.scss b/src/newsreader/scss/pages/import/components/index.scss new file mode 100644 index 0000000..0bccecf --- /dev/null +++ b/src/newsreader/scss/pages/import/components/index.scss @@ -0,0 +1 @@ +@import "import-form/index"; diff --git a/src/newsreader/scss/pages/import/elements/index.scss b/src/newsreader/scss/pages/import/elements/index.scss new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/scss/pages/import/index.scss b/src/newsreader/scss/pages/import/index.scss new file mode 100644 index 0000000..16b6493 --- /dev/null +++ b/src/newsreader/scss/pages/import/index.scss @@ -0,0 +1,8 @@ +// General imports +@import "../../partials/variables"; +@import "../../components/index"; +@import "../../elements/index"; + +// Page specific +@import "./components/index"; +@import "./elements/index"; diff --git a/src/newsreader/scss/pages/rules/components/card/_card.scss b/src/newsreader/scss/pages/rules/components/card/_card.scss index ec09189..efc2418 100644 --- a/src/newsreader/scss/pages/rules/components/card/_card.scss +++ b/src/newsreader/scss/pages/rules/components/card/_card.scss @@ -8,6 +8,10 @@ padding: 0 10px; } } + + &--action > .button { + margin: 0 10px; + } } &__content { diff --git a/src/newsreader/utils/formatter.sh b/src/newsreader/utils/formatter.sh deleted file mode 100644 index de70b4a..0000000 --- a/src/newsreader/utils/formatter.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -FILES=$(git diff --cached --name-only --diff-filter=ACM "*.py" | sed 's| |\\ |g') - -if [ ! -z "$FILES" ]; then - # Format all selected files - echo "$FILES" | xargs ./env/bin/isort - - # Add back the modified/prettified files to staging - echo "$FILES" | xargs git add -fi diff --git a/src/newsreader/utils/opml.py b/src/newsreader/utils/opml.py new file mode 100644 index 0000000..55a9387 --- /dev/null +++ b/src/newsreader/utils/opml.py @@ -0,0 +1,41 @@ +import logging + +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator + +from lxml.etree import XMLSyntaxError, parse + +from newsreader.news.collection.models import CollectionRule + + +def parse_opml(file, user, skip_existing=False): + known_urls = CollectionRule.objects.filter(user=user).values_list("url", flat=True) + + try: + tree = parse(file) + except XMLSyntaxError as e: + raise IOError("Invalid file") from e + + root = tree.getroot() + + validate = URLValidator(schemes=["http", "https"]) + + for element in root.iter(tag="outline"): + if not "xmlUrl" in element.keys(): + continue + + feed_url = element.get("xmlUrl") + name = element.get("text") + + if not all([feed_url, name]): + continue + elif skip_existing and feed_url in known_urls: + continue + + try: + validate(feed_url) + except ValidationError as e: + logging.info(f"Skipped due to invalid URL: {e}") + continue + + yield CollectionRule(url=feed_url, name=name, user=user) diff --git a/src/newsreader/utils/pre-commit b/src/newsreader/utils/pre-commit deleted file mode 100644 index d1e29b9..0000000 --- a/src/newsreader/utils/pre-commit +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Check if the directory is the root directory -if [ ! -d ".git/" ]; then - echo "Please commit from within the root directory" - exit 1 -fi - -# Run every file inside the pre-commit.d directory -for file in .git/hooks/pre-commit.d/* -do - . $file -done diff --git a/src/newsreader/utils/tests/__init__.py b/src/newsreader/utils/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/utils/tests/files/empty-feeds.opml b/src/newsreader/utils/tests/files/empty-feeds.opml new file mode 100644 index 0000000..34ad8f2 --- /dev/null +++ b/src/newsreader/utils/tests/files/empty-feeds.opml @@ -0,0 +1,9 @@ + + + + Sat, 18 Jan 2020 17:32:03 +0000 + Tiny Tiny RSS Feed Export + + + + diff --git a/src/newsreader/utils/tests/files/feeds.opml b/src/newsreader/utils/tests/files/feeds.opml new file mode 100644 index 0000000..2cc14cf --- /dev/null +++ b/src/newsreader/utils/tests/files/feeds.opml @@ -0,0 +1,15 @@ + + + + Sat, 18 Jan 2020 17:32:03 +0000 + Tiny Tiny RSS Feed Export + + + + + + + + + + diff --git a/src/newsreader/utils/tests/files/invalid-url-feeds.opml b/src/newsreader/utils/tests/files/invalid-url-feeds.opml new file mode 100644 index 0000000..aad9a7f --- /dev/null +++ b/src/newsreader/utils/tests/files/invalid-url-feeds.opml @@ -0,0 +1,16 @@ + + + + Sat, 18 Jan 2020 17:32:03 +0000 + Tiny Tiny RSS Feed Export + + + + + + + + + + + diff --git a/src/newsreader/utils/tests/files/missing-feeds.opml b/src/newsreader/utils/tests/files/missing-feeds.opml new file mode 100644 index 0000000..22ac9a3 --- /dev/null +++ b/src/newsreader/utils/tests/files/missing-feeds.opml @@ -0,0 +1,17 @@ + + + + Sat, 18 Jan 2020 17:32:03 +0000 + Tiny Tiny RSS Feed Export + + + + + + + + + + + + diff --git a/src/newsreader/utils/tests/files/test.png b/src/newsreader/utils/tests/files/test.png new file mode 100644 index 0000000..848a7ef Binary files /dev/null and b/src/newsreader/utils/tests/files/test.png differ diff --git a/src/newsreader/utils/tests/test_opml.py b/src/newsreader/utils/tests/test_opml.py new file mode 100644 index 0000000..d2baf8f --- /dev/null +++ b/src/newsreader/utils/tests/test_opml.py @@ -0,0 +1,46 @@ +import os + +from pathlib import Path + +from django.test import TestCase + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.utils.opml import parse_opml + + +class OPMImportTestCase(TestCase): + def setUp(self): + self.directory = Path(__file__).parent.absolute() + self.user = UserFactory() + + def test_simple(self): + path = os.path.join(self.directory, "files", "feeds.opml") + + with open(path, "r") as file: + rules = list(parse_opml(file, self.user)) + + self.assertEquals(len(rules), 4) + + def test_file_without_feeds(self): + path = os.path.join(self.directory, "files", "empty-feeds.opml") + + with open(path, "r") as file: + rules = list(parse_opml(file, self.user)) + + self.assertEquals(len(rules), 0) + + def test_file_with_missing_rule_properties(self): + path = os.path.join(self.directory, "files", "missing-feeds.opml") + + with open(path, "r") as file: + rules = list(parse_opml(file, self.user)) + + self.assertEquals(len(rules), 2) + + def test_url_validation(self): + path = os.path.join(self.directory, "files", "invalid-url-feeds.opml") + + with open(path, "r") as file: + rules = list(parse_opml(file, self.user)) + + self.assertEquals(len(rules), 0)