Check for Twitter error codes

Important for expired tokens as 401's are returned for various reasons

src/newsreader/news/collection/tests/twitter/client/tests.py

@@ -74,19 +74,24 @@ class TwitterClientTestCase(TestCase):
         self.mocked_read.assert_called()
 
     def test_client_catches_stream_denied_exception(self):
+        """
+        Twitter also returns these responses for accounts which have been shutdown.
+        Therefore the error codes should be checked inside the response body.
+        See https://stackoverflow.com/questions/8357568/do-twitter-access-token-expire
+        """
         user = UserFactory(
             twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
         )
         timeline = TwitterTimelineFactory(user=user)
 
-        self.mocked_read.side_effect = StreamDeniedException(message="Token expired")
+        self.mocked_read.side_effect = StreamDeniedException(message="Not authorized")
 
         with TwitterClient([timeline]) as client:
             for data, stream in client:
                 with self.subTest(data=data, stream=stream):
                     self.assertIsNone(data)
                     self.assertIsNone(stream)
-                    self.assertEquals(stream.rule.error, "Token expired")
+                    self.assertEquals(stream.rule.error, "Authorization required")
                     self.assertEquals(stream.rule.succeeded, False)
 
         self.mocked_read.assert_called()
@@ -94,8 +99,8 @@ class TwitterClientTestCase(TestCase):
         user.refresh_from_db()
         timeline.refresh_from_db()
 
-        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNotNone(user.twitter_oauth_token)
-        self.assertIsNone(user.twitter_oauth_token_secret)
+        self.assertIsNotNone(user.twitter_oauth_token_secret)
 
     def test_client_catches_stream_timed_out_exception(self):
         timeline = TwitterTimelineFactory()
@@ -160,3 +165,31 @@ class TwitterClientTestCase(TestCase):
                 self.assertEquals(stream.rule.succeeded, False)
 
         self.mocked_read.assert_called()
+
+    def test_client_catches_token_expired(self):
+        user = UserFactory(
+            twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
+        )
+        timeline = TwitterTimelineFactory(user=user)
+
+        response = Mock(json=lambda: {"errors": [{"code": 89}]})
+
+        self.mocked_read.side_effect = StreamDeniedException(
+            message="Not authorized", response=response
+        )
+
+        with TwitterClient([timeline]) as client:
+            for data, stream in client:
+                with self.subTest(data=data, stream=stream):
+                    self.assertIsNone(data)
+                    self.assertIsNone(stream)
+                    self.assertEquals(stream.rule.error, "Authorization required")
+                    self.assertEquals(stream.rule.succeeded, False)
+
+        self.mocked_read.assert_called()
+
+        user.refresh_from_db()
+        timeline.refresh_from_db()
+
+        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNone(user.twitter_oauth_token_secret)

src/newsreader/news/collection/tests/twitter/collector/tests.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 from uuid import uuid4
 
 from django.test import TestCase
@@ -152,8 +152,8 @@ class TwitterCollectorTestCase(TestCase):
 
         user = timeline.user
 
-        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNotNone(user.twitter_oauth_token)
-        self.assertIsNone(user.twitter_oauth_token_secret)
+        self.assertIsNotNone(user.twitter_oauth_token_secret)
 
     def test_forbidden(self):
         self.mocked_fetch.side_effect = StreamForbiddenException
@@ -178,3 +178,25 @@ class TwitterCollectorTestCase(TestCase):
         self.assertEquals(Post.objects.count(), 0)
         self.assertEquals(timeline.succeeded, False)
         self.assertEquals(timeline.error, "Stream timed out")
+
+    def test_token_expired(self):
+        response = Mock(json=lambda: {"errors": [{"code": 89}]})
+
+        self.mocked_fetch.side_effect = StreamDeniedException(response=response)
+
+        timeline = TwitterTimelineFactory(
+            user__twitter_oauth_token=str(uuid4()),
+            user__twitter_oauth_token_secret=str(uuid4()),
+        )
+
+        collector = TwitterCollector()
+        collector.collect(rules=[timeline])
+
+        self.assertEquals(Post.objects.count(), 0)
+        self.assertEquals(timeline.succeeded, False)
+        self.assertEquals(timeline.error, "Stream does not have sufficient permissions")
+
+        user = timeline.user
+
+        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNone(user.twitter_oauth_token_secret)

src/newsreader/news/collection/twitter.py

@@ -232,44 +232,6 @@ class TwitterClient(PostClient):
 
                     self.set_rule_error(stream.rule, e)
 
-                    break
-                except StreamDeniedException as e:
-                    logger.warning(
-                        f"Access token expired for user {stream.rule.user.pk}"
-                    )
-
-                    try:
-                        import sentry_sdk
-
-                        with sentry_sdk.push_scope() as scope:
-                            scope.set_extra(
-                                "content", e.response.content if e.response else None
-                            )
-                            sentry_sdk.capture_message(
-                                "Twitter authentication credentials reset"
-                            )
-                    except ImportError:
-                        pass
-
-                    stream.rule.user.twitter_oauth_token = None
-                    stream.rule.user.twitter_oauth_token_secret = None
-                    stream.rule.user.save()
-
-                    message = _(
-                        "Your Twitter account credentials have expired. Re-authenticate in"
-                        " the settings page to keep retrieving Twitter specific information"
-                        " from your account."
-                    )
-
-                    send_mail(
-                        "Twitter account needs re-authentication",
-                        message,
-                        None,
-                        [stream.rule.user.email],
-                    )
-
-                    self.set_rule_error(stream.rule, e)
-
                     break
                 except (StreamNotFoundException, StreamTimeOutException) as e:
                     logger.warning(f"Request failed for {stream.rule.screen_name}")
@@ -282,6 +244,46 @@ class TwitterClient(PostClient):
 
                     self.set_rule_error(stream.rule, e)
 
+                    if not e.response:
+                        continue
+
+                    try:
+                        response_data = e.response.json()
+                    except JSONDecodeError:
+                        continue
+
+                    if "errors" in response_data:
+                        errors = response_data["errors"]
+                        token_expired = any(error["code"] == 89 for error in errors)
+
+                        try:
+                            import sentry_sdk
+
+                            with sentry_sdk.push_scope() as scope:
+                                scope.set_extra("content", response_data)
+                                sentry_sdk.capture_message(
+                                    "Twitter authentication credentials reset"
+                                )
+                        except ImportError:
+                            pass
+
+                        stream.rule.user.twitter_oauth_token = None
+                        stream.rule.user.twitter_oauth_token_secret = None
+                        stream.rule.user.save()
+
+                        message = _(
+                            "Your Twitter account credentials have expired. Re-authenticate in"
+                            " the settings page to keep retrieving Twitter specific information"
+                            " from your account."
+                        )
+
+                        send_mail(
+                            "Twitter account needs re-authentication",
+                            message,
+                            None,
+                            [stream.rule.user.email],
+                        )
+
                     continue
                 finally:
                     stream.rule.last_run = timezone.now()