Check for Twitter error codes

Important for expired tokens as 401's are returned for various reasons

src/newsreader/news/collection/tests/twitter/client/tests.py

@@ -74,19 +74,24 @@ class TwitterClientTestCase(TestCase):
         self.mocked_read.assert_called()
 
     def test_client_catches_stream_denied_exception(self):
+        """
+        Twitter also returns these responses for accounts which have been shutdown.
+        Therefore the error codes should be checked inside the response body.
+        See https://stackoverflow.com/questions/8357568/do-twitter-access-token-expire
+        """
         user = UserFactory(
             twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
         )
         timeline = TwitterTimelineFactory(user=user)
 
-        self.mocked_read.side_effect = StreamDeniedException(message="Token expired")
+        self.mocked_read.side_effect = StreamDeniedException(message="Not authorized")
 
         with TwitterClient([timeline]) as client:
             for data, stream in client:
                 with self.subTest(data=data, stream=stream):
                     self.assertIsNone(data)
                     self.assertIsNone(stream)
-                    self.assertEquals(stream.rule.error, "Token expired")
+                    self.assertEquals(stream.rule.error, "Authorization required")
                     self.assertEquals(stream.rule.succeeded, False)
 
         self.mocked_read.assert_called()
@@ -94,8 +99,8 @@ class TwitterClientTestCase(TestCase):
         user.refresh_from_db()
         timeline.refresh_from_db()
 
-        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNotNone(user.twitter_oauth_token)
-        self.assertIsNone(user.twitter_oauth_token_secret)
+        self.assertIsNotNone(user.twitter_oauth_token_secret)
 
     def test_client_catches_stream_timed_out_exception(self):
         timeline = TwitterTimelineFactory()
@@ -160,3 +165,31 @@ class TwitterClientTestCase(TestCase):
                 self.assertEquals(stream.rule.succeeded, False)
 
         self.mocked_read.assert_called()
+
+    def test_client_catches_token_expired(self):
+        user = UserFactory(
+            twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
+        )
+        timeline = TwitterTimelineFactory(user=user)
+
+        response = Mock(json=lambda: {"errors": [{"code": 89}]})
+
+        self.mocked_read.side_effect = StreamDeniedException(
+            message="Not authorized", response=response
+        )
+
+        with TwitterClient([timeline]) as client:
+            for data, stream in client:
+                with self.subTest(data=data, stream=stream):
+                    self.assertIsNone(data)
+                    self.assertIsNone(stream)
+                    self.assertEquals(stream.rule.error, "Authorization required")
+                    self.assertEquals(stream.rule.succeeded, False)
+
+        self.mocked_read.assert_called()
+
+        user.refresh_from_db()
+        timeline.refresh_from_db()
+
+        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNone(user.twitter_oauth_token_secret)

src/newsreader/news/collection/tests/twitter/collector/tests.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 from uuid import uuid4
 
 from django.test import TestCase
@@ -152,8 +152,8 @@ class TwitterCollectorTestCase(TestCase):
 
         user = timeline.user
 
-        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNotNone(user.twitter_oauth_token)
-        self.assertIsNone(user.twitter_oauth_token_secret)
+        self.assertIsNotNone(user.twitter_oauth_token_secret)
 
     def test_forbidden(self):
         self.mocked_fetch.side_effect = StreamForbiddenException
@@ -178,3 +178,25 @@ class TwitterCollectorTestCase(TestCase):
         self.assertEquals(Post.objects.count(), 0)
         self.assertEquals(timeline.succeeded, False)
         self.assertEquals(timeline.error, "Stream timed out")
+
+    def test_token_expired(self):
+        response = Mock(json=lambda: {"errors": [{"code": 89}]})
+
+        self.mocked_fetch.side_effect = StreamDeniedException(response=response)
+
+        timeline = TwitterTimelineFactory(
+            user__twitter_oauth_token=str(uuid4()),
+            user__twitter_oauth_token_secret=str(uuid4()),
+        )
+
+        collector = TwitterCollector()
+        collector.collect(rules=[timeline])
+
+        self.assertEquals(Post.objects.count(), 0)
+        self.assertEquals(timeline.succeeded, False)
+        self.assertEquals(timeline.error, "Stream does not have sufficient permissions")
+
+        user = timeline.user
+
+        self.assertIsNone(user.twitter_oauth_token)
+        self.assertIsNone(user.twitter_oauth_token_secret)

src/newsreader/news/collection/twitter.py

@@ -233,18 +233,34 @@ class TwitterClient(PostClient):
                     self.set_rule_error(stream.rule, e)
 
                     break
-                except StreamDeniedException as e:
+                except (StreamNotFoundException, StreamTimeOutException) as e:
-                    logger.warning(
+                    logger.warning(f"Request failed for {stream.rule.screen_name}")
-                        f"Access token expired for user {stream.rule.user.pk}"
+
-                    )
+                    self.set_rule_error(stream.rule, e)
+
+                    continue
+                except StreamException as e:
+                    logger.exception(f"Request failed for {stream.rule.screen_name}")
+
+                    self.set_rule_error(stream.rule, e)
+
+                    if not e.response:
+                        continue
+
+                    try:
+                        response_data = e.response.json()
+                    except JSONDecodeError:
+                        continue
+
+                    if "errors" in response_data:
+                        errors = response_data["errors"]
+                        token_expired = any(error["code"] == 89 for error in errors)
 
                         try:
                             import sentry_sdk
 
                             with sentry_sdk.push_scope() as scope:
-                            scope.set_extra(
+                                scope.set_extra("content", response_data)
-                                "content", e.response.content if e.response else None
-                            )
                                 sentry_sdk.capture_message(
                                     "Twitter authentication credentials reset"
                                 )
@@ -268,20 +284,6 @@ class TwitterClient(PostClient):
                             [stream.rule.user.email],
                         )
 
-                    self.set_rule_error(stream.rule, e)
-
-                    break
-                except (StreamNotFoundException, StreamTimeOutException) as e:
-                    logger.warning(f"Request failed for {stream.rule.screen_name}")
-
-                    self.set_rule_error(stream.rule, e)
-
-                    continue
-                except StreamException as e:
-                    logger.exception(f"Request failed for {stream.rule.screen_name}")
-
-                    self.set_rule_error(stream.rule, e)
-
                     continue
                 finally:
                     stream.rule.last_run = timezone.now()