From 510f7187a86f84461cac78e31f4d8887c85861c0 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Tue, 27 Apr 2021 12:42:02 +0200 Subject: [PATCH] 0.3.13.7 --- CHANGELOG.md | 4 + package.json | 2 +- pyproject.toml | 2 +- .../collection/tests/twitter/client/tests.py | 41 +++++++++- .../tests/twitter/collector/tests.py | 28 ++++++- src/newsreader/news/collection/twitter.py | 78 ++++++++++--------- 6 files changed, 108 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6af3de9..507d857 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.3.13.7 + +- Check for Twitter error codes in response + ## 0.3.13.6 - Try to load sentry by default for all environments diff --git a/package.json b/package.json index 451e92a..aa50447 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "newsreader", - "version": "0.3.13.6", + "version": "0.3.13.7", "description": "Application for viewing RSS feeds", "main": "index.js", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index 700c6e3..a1f4385 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "newsreader" -version = "0.3.13.6" +version = "0.3.13.7" description = "Webapplication for reading RSS feeds" authors = ["Sonny "] license = "GPL-3.0" diff --git a/src/newsreader/news/collection/tests/twitter/client/tests.py b/src/newsreader/news/collection/tests/twitter/client/tests.py index 387ffef..9730d92 100644 --- a/src/newsreader/news/collection/tests/twitter/client/tests.py +++ b/src/newsreader/news/collection/tests/twitter/client/tests.py @@ -74,19 +74,24 @@ class TwitterClientTestCase(TestCase): self.mocked_read.assert_called() def test_client_catches_stream_denied_exception(self): + """ + Twitter also returns these responses for accounts which have been shutdown. + Therefore the error codes should be checked inside the response body. + See https://stackoverflow.com/questions/8357568/do-twitter-access-token-expire + """ user = UserFactory( twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4()) ) timeline = TwitterTimelineFactory(user=user) - self.mocked_read.side_effect = StreamDeniedException(message="Token expired") + self.mocked_read.side_effect = StreamDeniedException(message="Not authorized") with TwitterClient([timeline]) as client: for data, stream in client: with self.subTest(data=data, stream=stream): self.assertIsNone(data) self.assertIsNone(stream) - self.assertEquals(stream.rule.error, "Token expired") + self.assertEquals(stream.rule.error, "Authorization required") self.assertEquals(stream.rule.succeeded, False) self.mocked_read.assert_called() @@ -94,8 +99,8 @@ class TwitterClientTestCase(TestCase): user.refresh_from_db() timeline.refresh_from_db() - self.assertIsNone(user.twitter_oauth_token) - self.assertIsNone(user.twitter_oauth_token_secret) + self.assertIsNotNone(user.twitter_oauth_token) + self.assertIsNotNone(user.twitter_oauth_token_secret) def test_client_catches_stream_timed_out_exception(self): timeline = TwitterTimelineFactory() @@ -160,3 +165,31 @@ class TwitterClientTestCase(TestCase): self.assertEquals(stream.rule.succeeded, False) self.mocked_read.assert_called() + + def test_client_catches_token_expired(self): + user = UserFactory( + twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4()) + ) + timeline = TwitterTimelineFactory(user=user) + + response = Mock(json=lambda: {"errors": [{"code": 89}]}) + + self.mocked_read.side_effect = StreamDeniedException( + message="Not authorized", response=response + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Authorization required") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + user.refresh_from_db() + timeline.refresh_from_db() + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) diff --git a/src/newsreader/news/collection/tests/twitter/collector/tests.py b/src/newsreader/news/collection/tests/twitter/collector/tests.py index 766e971..ab589ff 100644 --- a/src/newsreader/news/collection/tests/twitter/collector/tests.py +++ b/src/newsreader/news/collection/tests/twitter/collector/tests.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import patch +from unittest.mock import Mock, patch from uuid import uuid4 from django.test import TestCase @@ -152,8 +152,8 @@ class TwitterCollectorTestCase(TestCase): user = timeline.user - self.assertIsNone(user.twitter_oauth_token) - self.assertIsNone(user.twitter_oauth_token_secret) + self.assertIsNotNone(user.twitter_oauth_token) + self.assertIsNotNone(user.twitter_oauth_token_secret) def test_forbidden(self): self.mocked_fetch.side_effect = StreamForbiddenException @@ -178,3 +178,25 @@ class TwitterCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 0) self.assertEquals(timeline.succeeded, False) self.assertEquals(timeline.error, "Stream timed out") + + def test_token_expired(self): + response = Mock(json=lambda: {"errors": [{"code": 89}]}) + + self.mocked_fetch.side_effect = StreamDeniedException(response=response) + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream does not have sufficient permissions") + + user = timeline.user + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 57cfe6c..2004fa1 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -232,44 +232,6 @@ class TwitterClient(PostClient): self.set_rule_error(stream.rule, e) - break - except StreamDeniedException as e: - logger.warning( - f"Access token expired for user {stream.rule.user.pk}" - ) - - try: - import sentry_sdk - - with sentry_sdk.push_scope() as scope: - scope.set_extra( - "content", e.response.content if e.response else None - ) - sentry_sdk.capture_message( - "Twitter authentication credentials reset" - ) - except ImportError: - pass - - stream.rule.user.twitter_oauth_token = None - stream.rule.user.twitter_oauth_token_secret = None - stream.rule.user.save() - - message = _( - "Your Twitter account credentials have expired. Re-authenticate in" - " the settings page to keep retrieving Twitter specific information" - " from your account." - ) - - send_mail( - "Twitter account needs re-authentication", - message, - None, - [stream.rule.user.email], - ) - - self.set_rule_error(stream.rule, e) - break except (StreamNotFoundException, StreamTimeOutException) as e: logger.warning(f"Request failed for {stream.rule.screen_name}") @@ -282,6 +244,46 @@ class TwitterClient(PostClient): self.set_rule_error(stream.rule, e) + if not e.response: + continue + + try: + response_data = e.response.json() + except JSONDecodeError: + continue + + if "errors" in response_data: + errors = response_data["errors"] + token_expired = any(error["code"] == 89 for error in errors) + + try: + import sentry_sdk + + with sentry_sdk.push_scope() as scope: + scope.set_extra("content", response_data) + sentry_sdk.capture_message( + "Twitter authentication credentials reset" + ) + except ImportError: + pass + + stream.rule.user.twitter_oauth_token = None + stream.rule.user.twitter_oauth_token_secret = None + stream.rule.user.save() + + message = _( + "Your Twitter account credentials have expired. Re-authenticate in" + " the settings page to keep retrieving Twitter specific information" + " from your account." + ) + + send_mail( + "Twitter account needs re-authentication", + message, + None, + [stream.rule.user.email], + ) + continue finally: stream.rule.last_run = timezone.now()