diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 22af39f..a306f17 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -219,12 +219,10 @@ class RedditBuilder(PostBuilder): class RedditStream(PostStream): rule_type = RuleTypeChoices.subreddit headers = {} - user = None def __init__(self, rule): super().__init__(rule) - self.user = self.rule.user self.headers = { f"Authorization": f"bearer {self.rule.user.reddit_access_token}" } diff --git a/src/newsreader/news/collection/tests/twitter/stream/mocks.py b/src/newsreader/news/collection/tests/twitter/stream/mocks.py index e69de29..1b7c6a2 100644 --- a/src/newsreader/news/collection/tests/twitter/stream/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/stream/mocks.py @@ -0,0 +1,225 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] diff --git a/src/newsreader/news/collection/tests/twitter/stream/tests.py b/src/newsreader/news/collection/tests/twitter/stream/tests.py index e69de29..4edb639 100644 --- a/src/newsreader/news/collection/tests/twitter/stream/tests.py +++ b/src/newsreader/news/collection/tests/twitter/stream/tests.py @@ -0,0 +1,107 @@ +from json import JSONDecodeError +from unittest.mock import patch + +from django.test import TestCase + +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamForbiddenException, + StreamNotFoundException, + StreamParseException, + StreamTimeOutException, + StreamTooManyException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.stream.mocks import simple_mock +from newsreader.news.collection.twitter import TwitterStream + + +class TwitterStreamTestCase(TestCase): + def setUp(self): + self.patched_fetch = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = self.patched_fetch.start() + + def tearDown(self): + patch.stopall() + + def test_simple_stream(self): + self.mocked_fetch.return_value.json.return_value = simple_mock + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + data, stream = stream.read() + + self.assertEquals(data, simple_mock) + self.assertEquals(stream, stream) + + self.mocked_fetch.assert_called() + + def test_stream_raises_exception(self): + self.mocked_fetch.side_effect = StreamException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_denied_exception(self): + self.mocked_fetch.side_effect = StreamDeniedException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamDeniedException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_not_found_exception(self): + self.mocked_fetch.side_effect = StreamNotFoundException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamNotFoundException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_time_out_exception(self): + self.mocked_fetch.side_effect = StreamTimeOutException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamTimeOutException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_forbidden_exception(self): + self.mocked_fetch.side_effect = StreamForbiddenException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamForbiddenException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_parse_exception(self): + self.mocked_fetch.return_value.json.side_effect = JSONDecodeError( + "No json found", "{}", 5 + ) + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamParseException): + stream.read() + + self.mocked_fetch.assert_called() diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 218c560..e2be78f 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -150,7 +150,27 @@ class TwitterBuilder(PostBuilder): class TwitterStream(PostStream): - pass + rule_type = RuleTypeChoices.subreddit + + def read(self): + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=self.rule.user.twitter_oauth_token, + resource_owner_secret=self.rule.user.twitter_oauth_token_secret, + ) + + response = fetch(self.rule.url, auth=oauth) + + return self.parse(response), self + + def parse(self, response): + try: + return response.json() + except JSONDecodeError as e: + raise StreamParseException( + response=response, message="Failed parsing json" + ) from e class TwitterClient(PostClient):