Show URL's during feed exceptions & use version number in User-Agent

src/newsreader/news/collection/feed.py

@@ -161,17 +161,27 @@ class FeedClient(Client):
                     stream.rule.last_suceeded = timezone.now()
 
                     yield response_data
+                except (StreamNotFoundException, StreamTimeOutException) as e:
+                    logger.warning(f"Request failed for {stream.rule.url}")
+
+                    self.set_rule_error(stream.rule, e)
+
+                    continue
                 except StreamException as e:
-                    logger.exception("Request failed")
+                    logger.exception(f"Request failed for {stream.rule.url}")
 
-                    length = stream.rule._meta.get_field("error").max_length
+                    self.set_rule_error(stream.rule, e)
-                    stream.rule.error = e.message[-length:]
-                    stream.rule.succeeded = False
 
-                    yield ({"entries": []}, stream)
+                    continue
                 finally:
                     stream.rule.save()
 
+    def set_rule_error(self, rule, exception):
+        length = rule._meta.get_field("error").max_length
+
+        rule.error = exception.message[-length:]
+        rule.succeeded = False
+
 
 class FeedCollector(Collector):
     builder = FeedBuilder

src/newsreader/news/collection/tests/feed/client/tests.py

@@ -41,13 +41,12 @@ class FeedClientTestCase(TestCase):
 
     def test_client_catches_stream_exception(self):
         rule = FeedFactory.create()
-        mock_stream = MagicMock(rule=rule)
 
         self.mocked_read.side_effect = StreamException(message="Stream exception")
 
         with FeedClient([rule]) as client:
             for data, stream in client:
-                self.assertEquals(data, {"entries": []})
+                self.assertEquals(data, None)
                 self.assertEquals(stream.rule.error, "Stream exception")
                 self.assertEquals(stream.rule.succeeded, False)
 
@@ -55,7 +54,6 @@ class FeedClientTestCase(TestCase):
 
     def test_client_catches_stream_not_found_exception(self):
         rule = FeedFactory.create()
-        mock_stream = MagicMock(rule=rule)
 
         self.mocked_read.side_effect = StreamNotFoundException(
             message="Stream not found"
@@ -63,7 +61,7 @@ class FeedClientTestCase(TestCase):
 
         with FeedClient([rule]) as client:
             for data, stream in client:
-                self.assertEquals(data, {"entries": []})
+                self.assertEquals(data, None)
                 self.assertEquals(stream.rule.error, "Stream not found")
                 self.assertEquals(stream.rule.succeeded, False)
 
@@ -71,13 +69,12 @@ class FeedClientTestCase(TestCase):
 
     def test_client_catches_stream_denied_exception(self):
         rule = FeedFactory.create()
-        mock_stream = MagicMock(rule=rule)
 
         self.mocked_read.side_effect = StreamDeniedException(message="Stream denied")
 
         with FeedClient([rule]) as client:
             for data, stream in client:
-                self.assertEquals(data, {"entries": []})
+                self.assertEquals(data, None)
                 self.assertEquals(stream.rule.error, "Stream denied")
                 self.assertEquals(stream.rule.succeeded, False)
 
@@ -85,7 +82,6 @@ class FeedClientTestCase(TestCase):
 
     def test_client_catches_stream_timed_out(self):
         rule = FeedFactory.create()
-        mock_stream = MagicMock(rule=rule)
 
         self.mocked_read.side_effect = StreamTimeOutException(
             message="Stream timed out"
@@ -93,7 +89,7 @@ class FeedClientTestCase(TestCase):
 
         with FeedClient([rule]) as client:
             for data, stream in client:
-                self.assertEquals(data, {"entries": []})
+                self.assertEquals(data, None)
                 self.assertEquals(stream.rule.error, "Stream timed out")
                 self.assertEquals(stream.rule.succeeded, False)
 
@@ -101,7 +97,6 @@ class FeedClientTestCase(TestCase):
 
     def test_client_catches_stream_parse_exception(self):
         rule = FeedFactory.create()
-        mock_stream = MagicMock(rule=rule)
 
         self.mocked_read.side_effect = StreamParseException(
             message="Stream has wrong contents"
@@ -109,7 +104,7 @@ class FeedClientTestCase(TestCase):
 
         with FeedClient([rule]) as client:
             for data, stream in client:
-                self.assertEquals(data, {"entries": []})
+                self.assertEquals(data, None)
                 self.assertEquals(stream.rule.error, "Stream has wrong contents")
                 self.assertEquals(stream.rule.succeeded, False)
 
@@ -117,13 +112,12 @@ class FeedClientTestCase(TestCase):
 
     def test_client_catches_long_exception_text(self):
         rule = FeedFactory.create()
-        mock_stream = MagicMock(rule=rule)
 
         self.mocked_read.side_effect = StreamParseException(message=words(1000))
 
         with FeedClient([rule]) as client:
             for data, stream in client:
-                self.assertEquals(data, {"entries": []})
+                self.assertEquals(data, None)
                 self.assertEquals(len(stream.rule.error), 1024)
                 self.assertEquals(stream.rule.succeeded, False)
 

src/newsreader/news/collection/utils.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+from django.conf import settings
 from django.db.models.fields import CharField, TextField
 from django.template.defaultfilters import truncatechars
 from django.utils import timezone
@@ -12,7 +13,7 @@ from requests.exceptions import RequestException
 from newsreader.news.collection.response_handler import ResponseHandler
 
 
-DEFAULT_HEADERS = {"User-Agent": "linux:rss.fudiggity.nl:v0.2"}
+DEFAULT_HEADERS = {"User-Agent": f"linux:rss.fudiggity.nl:{settings.VERSION}"}
 
 
 def build_publication_date(dt, tz):