diff --git a/src/newsreader/news/collection/feed.py b/src/newsreader/news/collection/feed.py index 8018bb5..f67a109 100644 --- a/src/newsreader/news/collection/feed.py +++ b/src/newsreader/news/collection/feed.py @@ -161,17 +161,27 @@ class FeedClient(Client): stream.rule.last_suceeded = timezone.now() yield response_data + except (StreamNotFoundException, StreamTimeOutException) as e: + logger.warning(f"Request failed for {stream.rule.url}") + + self.set_rule_error(stream.rule, e) + + continue except StreamException as e: - logger.exception("Request failed") + logger.exception(f"Request failed for {stream.rule.url}") - length = stream.rule._meta.get_field("error").max_length - stream.rule.error = e.message[-length:] - stream.rule.succeeded = False + self.set_rule_error(stream.rule, e) - yield ({"entries": []}, stream) + continue finally: stream.rule.save() + def set_rule_error(self, rule, exception): + length = rule._meta.get_field("error").max_length + + rule.error = exception.message[-length:] + rule.succeeded = False + class FeedCollector(Collector): builder = FeedBuilder diff --git a/src/newsreader/news/collection/tests/feed/client/tests.py b/src/newsreader/news/collection/tests/feed/client/tests.py index 59b5f65..24eb214 100644 --- a/src/newsreader/news/collection/tests/feed/client/tests.py +++ b/src/newsreader/news/collection/tests/feed/client/tests.py @@ -41,13 +41,12 @@ class FeedClientTestCase(TestCase): def test_client_catches_stream_exception(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) self.mocked_read.side_effect = StreamException(message="Stream exception") with FeedClient([rule]) as client: for data, stream in client: - self.assertEquals(data, {"entries": []}) + self.assertEquals(data, None) self.assertEquals(stream.rule.error, "Stream exception") self.assertEquals(stream.rule.succeeded, False) @@ -55,7 +54,6 @@ class FeedClientTestCase(TestCase): def test_client_catches_stream_not_found_exception(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) self.mocked_read.side_effect = StreamNotFoundException( message="Stream not found" @@ -63,7 +61,7 @@ class FeedClientTestCase(TestCase): with FeedClient([rule]) as client: for data, stream in client: - self.assertEquals(data, {"entries": []}) + self.assertEquals(data, None) self.assertEquals(stream.rule.error, "Stream not found") self.assertEquals(stream.rule.succeeded, False) @@ -71,13 +69,12 @@ class FeedClientTestCase(TestCase): def test_client_catches_stream_denied_exception(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) self.mocked_read.side_effect = StreamDeniedException(message="Stream denied") with FeedClient([rule]) as client: for data, stream in client: - self.assertEquals(data, {"entries": []}) + self.assertEquals(data, None) self.assertEquals(stream.rule.error, "Stream denied") self.assertEquals(stream.rule.succeeded, False) @@ -85,7 +82,6 @@ class FeedClientTestCase(TestCase): def test_client_catches_stream_timed_out(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) self.mocked_read.side_effect = StreamTimeOutException( message="Stream timed out" @@ -93,7 +89,7 @@ class FeedClientTestCase(TestCase): with FeedClient([rule]) as client: for data, stream in client: - self.assertEquals(data, {"entries": []}) + self.assertEquals(data, None) self.assertEquals(stream.rule.error, "Stream timed out") self.assertEquals(stream.rule.succeeded, False) @@ -101,7 +97,6 @@ class FeedClientTestCase(TestCase): def test_client_catches_stream_parse_exception(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) self.mocked_read.side_effect = StreamParseException( message="Stream has wrong contents" @@ -109,7 +104,7 @@ class FeedClientTestCase(TestCase): with FeedClient([rule]) as client: for data, stream in client: - self.assertEquals(data, {"entries": []}) + self.assertEquals(data, None) self.assertEquals(stream.rule.error, "Stream has wrong contents") self.assertEquals(stream.rule.succeeded, False) @@ -117,13 +112,12 @@ class FeedClientTestCase(TestCase): def test_client_catches_long_exception_text(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) self.mocked_read.side_effect = StreamParseException(message=words(1000)) with FeedClient([rule]) as client: for data, stream in client: - self.assertEquals(data, {"entries": []}) + self.assertEquals(data, None) self.assertEquals(len(stream.rule.error), 1024) self.assertEquals(stream.rule.succeeded, False) diff --git a/src/newsreader/news/collection/utils.py b/src/newsreader/news/collection/utils.py index 8ba6fec..d47cd68 100644 --- a/src/newsreader/news/collection/utils.py +++ b/src/newsreader/news/collection/utils.py @@ -1,5 +1,6 @@ from datetime import datetime +from django.conf import settings from django.db.models.fields import CharField, TextField from django.template.defaultfilters import truncatechars from django.utils import timezone @@ -12,7 +13,7 @@ from requests.exceptions import RequestException from newsreader.news.collection.response_handler import ResponseHandler -DEFAULT_HEADERS = {"User-Agent": "linux:rss.fudiggity.nl:v0.2"} +DEFAULT_HEADERS = {"User-Agent": f"linux:rss.fudiggity.nl:{settings.VERSION}"} def build_publication_date(dt, tz):