Update twitter error handling

src/newsreader/news/collection/tests/twitter/client/tests.py

@@ -193,3 +193,35 @@ class TwitterClientTestCase(TestCase):
 
         self.assertIsNone(user.twitter_oauth_token)
         self.assertIsNone(user.twitter_oauth_token_secret)
+
+    def test_client_does_not_reset_token(self):
+        """
+        The user's token and refresh token should not be reset when an generic
+        exception is caught
+        """
+        user = UserFactory(
+            twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
+        )
+        timeline = TwitterTimelineFactory(user=user)
+
+        response = Mock(json=lambda: {"errors": [{"code": 100}]})
+
+        self.mocked_read.side_effect = StreamException(
+            message="Generic message", response=response
+        )
+
+        with TwitterClient([timeline]) as client:
+            for data, stream in client:
+                with self.subTest(data=data, stream=stream):
+                    self.assertIsNone(data)
+                    self.assertIsNone(stream)
+                    self.assertEquals(stream.rule.error, "")
+                    self.assertEquals(stream.rule.succeeded, False)
+
+        self.mocked_read.assert_called()
+
+        user.refresh_from_db()
+        timeline.refresh_from_db()
+
+        self.assertIsNotNone(user.twitter_oauth_token)
+        self.assertIsNotNone(user.twitter_oauth_token_secret)

src/newsreader/news/collection/twitter.py

@@ -250,12 +250,14 @@ class TwitterClient(PostClient):
                     try:
                         response_data = e.response.json()
                     except JSONDecodeError:
+                        logger.exception("Could not parse json for request")
                         continue
 
                     if "errors" in response_data:
                         errors = response_data["errors"]
                         token_expired = any(error["code"] == 89 for error in errors)
 
+                        if token_expired:
                             try:
                                 import sentry_sdk