From a1ebbe68f5b9e1eb53f9fc7caee360eef41345c5 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 22:35:45 +0200 Subject: [PATCH] Add TwitterCollector tests --- .../tests/twitter/collector/mocks.py | 227 ++++++++++++++++++ .../tests/twitter/collector/tests.py | 180 ++++++++++++++ src/newsreader/news/collection/twitter.py | 2 +- 3 files changed, 408 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/tests/twitter/collector/mocks.py b/src/newsreader/news/collection/tests/twitter/collector/mocks.py index e69de29..c57f9cf 100644 --- a/src/newsreader/news/collection/tests/twitter/collector/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/collector/mocks.py @@ -0,0 +1,227 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +empty_mock = [] diff --git a/src/newsreader/news/collection/tests/twitter/collector/tests.py b/src/newsreader/news/collection/tests/twitter/collector/tests.py index e69de29..cf4e3ea 100644 --- a/src/newsreader/news/collection/tests/twitter/collector/tests.py +++ b/src/newsreader/news/collection/tests/twitter/collector/tests.py @@ -0,0 +1,180 @@ +from datetime import datetime +from unittest.mock import patch +from uuid import uuid4 + +from django.test import TestCase +from django.utils import timezone + +import pytz + +from freezegun import freeze_time +from ftfy import fix_text + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamForbiddenException, + StreamNotFoundException, + StreamTimeOutException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.collector.mocks import ( + empty_mock, + simple_mock, +) +from newsreader.news.collection.twitter import TWITTER_URL, TwitterCollector +from newsreader.news.collection.utils import truncate_text +from newsreader.news.core.models import Post + + +@freeze_time("2020-09-26 14:40:00") +class TwitterCollectorTestCase(TestCase): + def setUp(self): + patched_get = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = patched_get.start() + + patched_parse = patch("newsreader.news.collection.twitter.TwitterStream.parse") + self.mocked_parse = patched_parse.start() + + def tearDown(self): + patch.stopall() + + def test_simple_batch(self): + self.mocked_parse.return_value = simple_mock + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + screen_name="RobertsSpaceInd", + enabled=True, + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertCountEqual( + Post.objects.values_list("remote_identifier", flat=True), + ("1307054882210435074", "1307029168941461504"), + ) + + self.assertEquals(timeline.succeeded, True) + self.assertEquals(timeline.last_run, timezone.now()) + self.assertIsNone(timeline.error) + + post = Post.objects.get( + remote_identifier="1307054882210435074", + rule__type=RuleTypeChoices.twitter_timeline, + ) + + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 20, 32, 22)) + ) + + title = truncate_text( + Post, + "title", + "It's a close match-up for #SCShipShowdown today! Which Aegis ship" + " do you think will make it to the Semi-Finals?", + ) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals(post.title, title) + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307054882210435074" + ) + + post = Post.objects.get( + remote_identifier="1307029168941461504", + rule__type=RuleTypeChoices.twitter_timeline, + ) + + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 18, 50, 11)) + ) + + body = fix_text( + "We\u2019re welcoming members of our Builds, Publishes and Platform" + " teams on Star Citizen Live to talk about the process involved in" + " bringing everyone\u2019s work together and getting it out into your" + " hands. Going live on #Twitch in 10 minutes." + " \ud83c\udfa5\ud83d\udd34 \n\nTune in:" + " https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9" + ) + + title = truncate_text(Post, "title", body) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals(post.title, title) + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307029168941461504" + ) + + def test_empty_batch(self): + self.mocked_parse.return_value = empty_mock + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + + self.assertEquals(timeline.succeeded, True) + self.assertEquals(timeline.last_run, timezone.now()) + self.assertIsNone(timeline.error) + + def test_not_found(self): + self.mocked_fetch.side_effect = StreamNotFoundException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream not found") + + def test_denied(self): + self.mocked_fetch.side_effect = StreamDeniedException + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream does not have sufficient permissions") + + user = timeline.user + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) + + def test_forbidden(self): + self.mocked_fetch.side_effect = StreamForbiddenException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream forbidden") + + def test_timed_out(self): + self.mocked_fetch.side_effect = StreamTimeOutException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream timed out") diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 7e09c69..0824532 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -154,7 +154,7 @@ class TwitterBuilder(PostBuilder): class TwitterStream(PostStream): - rule_type = RuleTypeChoices.subreddit + rule_type = RuleTypeChoices.twitter_timeline def read(self): oauth = OAuth(