[#14] opml import export

gulp/sass.js

@@ -28,6 +28,7 @@ const taskMappings = [
   { name: 'category', destDir: `${CORE_DIR}/${STATIC_SUFFIX}` },
   { name: 'rules', destDir: `${COLLECTION_DIR}/${STATIC_SUFFIX}` },
   { name: 'rule', destDir: `${COLLECTION_DIR}/${STATIC_SUFFIX}` },
+  { name: 'import', destDir: `${COLLECTION_DIR}/${STATIC_SUFFIX}` },
 ];
 
 export const sassTask = done => {

src/newsreader/conf/base.py

@@ -12,9 +12,10 @@ https://docs.djangoproject.com/en/2.2/ref/settings/
 
 import os
 
+from pathlib import Path
 
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BASE_DIR = Path(__file__).resolve().parent.parent
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/

src/newsreader/js/pages/rules/App.js

@@ -74,9 +74,15 @@ class App extends React.Component {
     const pageHeader = (
       <>
         <h1 className="h1">Rules</h1>
-        <a className="link button button--confirm" href="/rules/create/">
+
-          Create rule
+        <div className="card__header--action">
-        </a>
+          <a className="link button button--primary" href="/rules/import/">
+            Import rules
+          </a>
+          <a className="link button button--confirm" href="/rules/create/">
+            Create rule
+          </a>
+        </div>
       </>
     );
 

src/newsreader/news/collection/forms.py

@@ -28,3 +28,8 @@ class CollectionRuleForm(forms.ModelForm):
     class Meta:
         model = CollectionRule
         fields = ("name", "url", "timezone", "favicon", "category")
+
+
+class OPMLImportForm(forms.Form):
+    file = forms.FileField(allow_empty_file=False)
+    skip_existing = forms.BooleanField(initial=False, required=False)

src/newsreader/news/collection/templates/collection/import.html

@@ -0,0 +1,41 @@
+{% extends "base.html" %}
+
+{% load static i18n %}
+
+{% block head %}
+  <link href="{% static 'collection/dist/css/import.css' %}" rel="stylesheet" />
+{% endblock %}
+
+{% block content %}
+  <div class="content">
+    <form class="form import-form" method="post" enctype="multipart/form-data">
+      {% csrf_token %}
+      {{ form.non_field_errors }}
+
+      <div class="form__header">
+        <h1 class="h1 form__title">{% trans "Import an OPML file" %}</h1>
+      </div>
+      <section class="section form__section import-form__section">
+        <fieldset class="form__fieldset import-form__fieldset">
+          <label class="label import-form__label" for="name">
+            {% trans "File" %}
+          </label>
+          {{ form.file.errors }}
+          {{ form.file }}
+        </fieldset>
+
+        <fieldset class="form__fieldset import-form__fieldset">
+          <label class="label import-form__label" for="name">
+            {% trans "Skip existing" %}
+          </label>
+          {{ form.skip_existing }}
+        </fieldset>
+
+        <fieldset class="form__fieldset import-form__fieldset">
+          <a class="link button button--cancel" href="{% url 'rules' %}">Cancel</a>
+          <button class="button button--confirm">Import</button>
+        </fieldset>
+      </section>
+    </form>
+  </div>
+{% endblock %}

src/newsreader/news/collection/tests/test_views.py

@@ -1,5 +1,9 @@
+import os
+
+from django.conf import settings
 from django.test import Client, TestCase
 from django.urls import reverse
+from django.utils.translation import gettext_lazy as _
 
 import pytz
 
@@ -146,3 +150,131 @@ class CollectionRuleUpdateViewTestCase(CollectionRuleViewTestCase, TestCase):
         self.rule.refresh_from_db()
 
         self.assertEquals(self.rule.category, None)
+
+
+class OPMLImportTestCase(TestCase):
+    def setUp(self):
+        self.user = UserFactory(password="test")
+        self.client.login(email=self.user.email, password="test")
+
+        self.form_data = {"file": "", "skip_existing": False}
+        self.url = reverse("import")
+
+    def _get_file_path(self, name):
+        file_dir = os.path.join(settings.BASE_DIR, "utils", "tests", "files")
+        return os.path.join(file_dir, name)
+
+    def test_simple(self):
+        file_path = self._get_file_path("feeds.opml")
+
+        with open(file_path) as file:
+            self.form_data.update(file=file)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertRedirects(response, reverse("rules"))
+
+        rules = CollectionRule.objects.all()
+        self.assertEquals(len(rules), 4)
+
+    def test_existing_rules(self):
+        CollectionRuleFactory(url="http://www.engadget.com/rss-full.xml", user=self.user)
+        CollectionRuleFactory(url="https://news.ycombinator.com/rss", user=self.user)
+        CollectionRuleFactory(
+            url="http://feeds.feedburner.com/Techcrunch", user=self.user
+        )
+        CollectionRuleFactory(
+            url="http://feeds.feedburner.com/tweakers/nieuws", user=self.user
+        )
+
+        file_path = self._get_file_path("feeds.opml")
+
+        with open(file_path) as file:
+            self.form_data.update(file=file)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertRedirects(response, reverse("rules"))
+
+        rules = CollectionRule.objects.all()
+        self.assertEquals(len(rules), 8)
+
+    def test_skip_existing_rules(self):
+        CollectionRuleFactory(url="http://www.engadget.com/rss-full.xml", user=self.user)
+        CollectionRuleFactory(url="https://news.ycombinator.com/rss", user=self.user)
+        CollectionRuleFactory(
+            url="http://feeds.feedburner.com/Techcrunch", user=self.user
+        )
+        CollectionRuleFactory(
+            url="http://feeds.feedburner.com/tweakers/nieuws", user=self.user
+        )
+
+        file_path = self._get_file_path("feeds.opml")
+
+        with open(file_path) as file:
+            self.form_data.update(file=file, skip_existing=True)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertEquals(response.status_code, 200)
+
+        rules = CollectionRule.objects.all()
+        self.assertEquals(len(rules), 4)
+
+    def test_empty_feed_file(self):
+        file_path = self._get_file_path("empty-feeds.opml")
+
+        with open(file_path) as file:
+            self.form_data.update(file=file)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertEquals(response.status_code, 200)
+
+        rules = CollectionRule.objects.all()
+        self.assertEquals(len(rules), 0)
+
+        self.assertFormError(response, "form", "file", _("No (new) rules found"))
+
+    def test_invalid_feeds(self):
+        file_path = self._get_file_path("invalid-url-feeds.opml")
+
+        with open(file_path) as file:
+            self.form_data.update(file=file)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertEquals(response.status_code, 200)
+
+        rules = CollectionRule.objects.all()
+
+        self.assertEquals(len(rules), 0)
+        self.assertFormError(response, "form", "file", _("No (new) rules found"))
+
+    def test_invalid_file(self):
+        file_path = self._get_file_path("test.png")
+
+        with open(file_path, "rb") as file:
+            self.form_data.update(file=file)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertEquals(response.status_code, 200)
+
+        rules = CollectionRule.objects.all()
+        self.assertEquals(len(rules), 0)
+
+        self.assertFormError(response, "form", "file", _("Invalid OPML file"))
+
+    def test_feeds_with_missing_attr(self):
+        file_path = self._get_file_path("missing-feeds.opml")
+
+        with open(file_path) as file:
+            self.form_data.update(file=file)
+
+            response = self.client.post(self.url, self.form_data)
+
+        self.assertRedirects(response, reverse("rules"))
+
+        rules = CollectionRule.objects.all()
+        self.assertEquals(len(rules), 2)

src/newsreader/news/collection/urls.py

@@ -11,6 +11,7 @@ from newsreader.news.collection.views import (
     CollectionRuleCreateView,
     CollectionRuleListView,
     CollectionRuleUpdateView,
+    OPMLImportView,
 )
 
 
@@ -33,4 +34,5 @@ urlpatterns = [
         login_required(CollectionRuleCreateView.as_view()),
         name="rule-create",
     ),
+    path("rules/import/", login_required(OPMLImportView.as_view()), name="import"),
 ]

src/newsreader/news/collection/views.py

@@ -1,14 +1,17 @@
 from typing import Dict, Iterable
 
+from django.contrib import messages
 from django.urls import reverse_lazy
-from django.views.generic.edit import CreateView, UpdateView
+from django.utils.translation import gettext_lazy as _
+from django.views.generic.edit import CreateView, FormView, UpdateView
 from django.views.generic.list import ListView
 
 import pytz
 
-from newsreader.news.collection.forms import CollectionRuleForm
+from newsreader.news.collection.forms import CollectionRuleForm, OPMLImportForm
 from newsreader.news.collection.models import CollectionRule
 from newsreader.news.core.models import Category
+from newsreader.utils.opml import parse_opml
 
 
 class CollectionRuleViewMixin:
@@ -56,3 +59,31 @@ class CollectionRuleCreateView(
     CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView
 ):
     template_name = "collection/rule-create.html"
+
+
+class OPMLImportView(FormView):
+    form_class = OPMLImportForm
+    success_url = reverse_lazy("rules")
+    template_name = "collection/import.html"
+
+    def form_valid(self, form):
+        user = self.request.user
+        file = form.cleaned_data["file"]
+        skip_existing = form.cleaned_data["skip_existing"]
+
+        instances = parse_opml(file, user, skip_existing=skip_existing)
+
+        try:
+            rules = CollectionRule.objects.bulk_create(instances)
+        except IOError:
+            form.add_error("file", _("Invalid OPML file"))
+            return self.form_invalid(form)
+
+        if not rules:
+            form.add_error("file", _("No (new) rules found"))
+            return self.form_invalid(form)
+
+        message = _(f"{len(rules)} new rules created")
+        messages.success(self.request, message)
+
+        return super().form_valid(form)

src/newsreader/scss/pages/import/components/import-form/_import-form.scss

@@ -0,0 +1,17 @@
+.import-form {
+  margin: 20px 0;
+
+  &__fieldset:last-child {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+  }
+
+  & input[type=file] {
+    width: 50%;
+  }
+
+  & input[type=checkbox] {
+    margin: 0 auto 0 10px;
+  }
+}

src/newsreader/scss/pages/import/components/import-form/index.scss

@@ -0,0 +1 @@
+@import "import-form";

src/newsreader/scss/pages/import/components/index.scss

@@ -0,0 +1 @@
+@import "import-form/index";

src/newsreader/scss/pages/import/index.scss

@@ -0,0 +1,8 @@
+// General imports
+@import "../../partials/variables";
+@import "../../components/index";
+@import "../../elements/index";
+
+// Page specific
+@import "./components/index";
+@import "./elements/index";

src/newsreader/scss/pages/rules/components/card/_card.scss

@@ -8,6 +8,10 @@
         padding: 0 10px;
       }
     }
+
+    &--action > .button {
+      margin: 0 10px;
+    }
   }
 
   &__content {

src/newsreader/utils/formatter.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-FILES=$(git diff --cached --name-only --diff-filter=ACM "*.py" | sed 's| |\\ |g')
-
-if [ ! -z "$FILES" ]; then
-    # Format all selected files
-    echo "$FILES" | xargs ./env/bin/isort
-
-    # Add back the modified/prettified files to staging
-    echo "$FILES" | xargs git add
-fi

src/newsreader/utils/opml.py

@@ -0,0 +1,41 @@
+import logging
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+
+from lxml.etree import XMLSyntaxError, parse
+
+from newsreader.news.collection.models import CollectionRule
+
+
+def parse_opml(file, user, skip_existing=False):
+    known_urls = CollectionRule.objects.filter(user=user).values_list("url", flat=True)
+
+    try:
+        tree = parse(file)
+    except XMLSyntaxError as e:
+        raise IOError("Invalid file") from e
+
+    root = tree.getroot()
+
+    validate = URLValidator(schemes=["http", "https"])
+
+    for element in root.iter(tag="outline"):
+        if not "xmlUrl" in element.keys():
+            continue
+
+        feed_url = element.get("xmlUrl")
+        name = element.get("text")
+
+        if not all([feed_url, name]):
+            continue
+        elif skip_existing and feed_url in known_urls:
+            continue
+
+        try:
+            validate(feed_url)
+        except ValidationError as e:
+            logging.info(f"Skipped due to invalid URL: {e}")
+            continue
+
+        yield CollectionRule(url=feed_url, name=name, user=user)

src/newsreader/utils/pre-commit

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Check if the directory is the root directory
-if [ ! -d ".git/" ]; then
-    echo "Please commit from within the root directory"
-    exit 1
-fi
-
-# Run every file inside the pre-commit.d directory
-for file in .git/hooks/pre-commit.d/*
-do
-    . $file
-done

src/newsreader/utils/tests/files/empty-feeds.opml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<opml version="1.0">
+  <head>
+    <dateCreated>Sat, 18 Jan 2020 17:32:03 +0000</dateCreated>
+    <title>Tiny Tiny RSS Feed Export</title>
+  </head>
+  <body>
+  </body>
+</opml>

src/newsreader/utils/tests/files/feeds.opml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<opml version="1.0">
+  <head>
+    <dateCreated>Sat, 18 Jan 2020 17:32:03 +0000</dateCreated>
+    <title>Tiny Tiny RSS Feed Export</title>
+  </head>
+  <body>
+    <outline text="Tech" ttrssSortOrder="0">
+      <outline type="rss" text="Engadget" xmlUrl="http://www.engadget.com/rss-full.xml" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="http://www.engadget.com"/>
+      <outline type="rss" text="Hacker News" xmlUrl="https://news.ycombinator.com/rss" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://news.ycombinator.com/"/>
+      <outline type="rss" text="TechCrunch" xmlUrl="http://feeds.feedburner.com/Techcrunch" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="http://techcrunch.com"/>
+      <outline type="rss" text="Tweakers" xmlUrl="http://feeds.feedburner.com/tweakers/nieuws" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="http://tweakers.net/nieuws"/>
+    </outline>
+  </body>
+</opml>

src/newsreader/utils/tests/files/invalid-url-feeds.opml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<opml version="1.0">
+  <head>
+    <dateCreated>Sat, 18 Jan 2020 17:32:03 +0000</dateCreated>
+    <title>Tiny Tiny RSS Feed Export</title>
+  </head>
+  <body>
+    <outline text="Buitenlands nieuws" ttrssSortOrder="0">
+      <outline type="rss" text="The Guardian" xmlUrl="ftp://www.theguardian.com/international/rss" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://www.theguardian.com/international"/>
+    </outline>
+    <outline text="Gaming" ttrssSortOrder="0">
+      <outline type="rss" text="Cohhcarnage" xmlUrl="twitrss/twitter_user_to_rss/?user=cohhcarnage" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://twitter.com/cohhcarnage"/>
+      <outline type="rss" text="dansgaming" xmlUrl="htp://twitrss.me/twitter_user_to_rss/?user=dansgaming" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://twitter.com/dansgaming"/>
+    </outline>
+  </body>
+</opml>

src/newsreader/utils/tests/files/missing-feeds.opml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<opml version="1.0">
+  <head>
+    <dateCreated>Sat, 18 Jan 2020 17:32:03 +0000</dateCreated>
+    <title>Tiny Tiny RSS Feed Export</title>
+  </head>
+  <body>
+    <outline text="Buitenlands nieuws" ttrssSortOrder="0">
+      <outline type="rss" text="BBC News - Home" xmlUrl="http://feeds.bbci.co.uk/news/rss.xml" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="http://www.bbc.co.uk/news/"/>
+      <outline type="rss" text="The Guardian" xmlUrl="" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://www.theguardian.com/international"/>
+    </outline>
+    <outline text="Gaming" ttrssSortOrder="0">
+      <outline type="rss" text="Cohhcarnage" xmlUrl="http://twitrss.me/twitter_user_to_rss/?user=cohhcarnage" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://twitter.com/cohhcarnage"/>
+      <outline type="rss" text="" xmlUrl="http://twitrss.me/twitter_user_to_rss/?user=dansgaming" ttrssSortOrder="0" ttrssUpdateInterval="0" htmlUrl="https://twitter.com/dansgaming"/>
+    </outline>
+  </body>
+</opml>

src/newsreader/utils/tests/test_opml.py

@@ -0,0 +1,46 @@
+import os
+
+from pathlib import Path
+
+from django.test import TestCase
+
+from newsreader.accounts.tests.factories import UserFactory
+from newsreader.utils.opml import parse_opml
+
+
+class OPMImportTestCase(TestCase):
+    def setUp(self):
+        self.directory = Path(__file__).parent.absolute()
+        self.user = UserFactory()
+
+    def test_simple(self):
+        path = os.path.join(self.directory, "files", "feeds.opml")
+
+        with open(path, "r") as file:
+            rules = list(parse_opml(file, self.user))
+
+            self.assertEquals(len(rules), 4)
+
+    def test_file_without_feeds(self):
+        path = os.path.join(self.directory, "files", "empty-feeds.opml")
+
+        with open(path, "r") as file:
+            rules = list(parse_opml(file, self.user))
+
+            self.assertEquals(len(rules), 0)
+
+    def test_file_with_missing_rule_properties(self):
+        path = os.path.join(self.directory, "files", "missing-feeds.opml")
+
+        with open(path, "r") as file:
+            rules = list(parse_opml(file, self.user))
+
+            self.assertEquals(len(rules), 2)
+
+    def test_url_validation(self):
+        path = os.path.join(self.directory, "files", "invalid-url-feeds.opml")
+
+        with open(path, "r") as file:
+            rules = list(parse_opml(file, self.user))
+
+            self.assertEquals(len(rules), 0)