Add TwitterCollector tests
This commit is contained in:
parent
7745d29b25
commit
a1ebbe68f5
3 changed files with 408 additions and 1 deletions
|
|
@ -0,0 +1,227 @@
|
|||
# retrieved with:
|
||||
# curl -X GET -H "Authorization: Bearer <TOKEN>" "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys
|
||||
|
||||
simple_mock = [
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 20:32:22 +0000 2020",
|
||||
"display_text_range": [0, 111],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}],
|
||||
"symbols": [],
|
||||
"urls": [],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"favorite_count": 54,
|
||||
"favorited": False,
|
||||
"full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?",
|
||||
"geo": None,
|
||||
"id": 1307054882210435074,
|
||||
"id_str": "1307054882210435074",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"retweet_count": 9,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 18:50:11 +0000 2020",
|
||||
"display_text_range": [0, 271],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [211, 218], "text": "Twitch"}],
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
],
|
||||
"symbols": [],
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "twitch.tv/starcitizen",
|
||||
"expanded_url": "http://twitch.tv/starcitizen",
|
||||
"indices": [248, 271],
|
||||
"url": "https://t.co/2AdNovhpFW",
|
||||
}
|
||||
],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"extended_entities": {
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
]
|
||||
},
|
||||
"favorite_count": 90,
|
||||
"favorited": False,
|
||||
"full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9",
|
||||
"geo": None,
|
||||
"id": 1307029168941461504,
|
||||
"id_str": "1307029168941461504",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"possibly_sensitive": False,
|
||||
"retweet_count": 13,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
empty_mock = []
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
import pytz
|
||||
|
||||
from freezegun import freeze_time
|
||||
from ftfy import fix_text
|
||||
|
||||
from newsreader.news.collection.choices import RuleTypeChoices
|
||||
from newsreader.news.collection.exceptions import (
|
||||
StreamDeniedException,
|
||||
StreamForbiddenException,
|
||||
StreamNotFoundException,
|
||||
StreamTimeOutException,
|
||||
)
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.tests.twitter.collector.mocks import (
|
||||
empty_mock,
|
||||
simple_mock,
|
||||
)
|
||||
from newsreader.news.collection.twitter import TWITTER_URL, TwitterCollector
|
||||
from newsreader.news.collection.utils import truncate_text
|
||||
from newsreader.news.core.models import Post
|
||||
|
||||
|
||||
@freeze_time("2020-09-26 14:40:00")
|
||||
class TwitterCollectorTestCase(TestCase):
|
||||
def setUp(self):
|
||||
patched_get = patch("newsreader.news.collection.twitter.fetch")
|
||||
self.mocked_fetch = patched_get.start()
|
||||
|
||||
patched_parse = patch("newsreader.news.collection.twitter.TwitterStream.parse")
|
||||
self.mocked_parse = patched_parse.start()
|
||||
|
||||
def tearDown(self):
|
||||
patch.stopall()
|
||||
|
||||
def test_simple_batch(self):
|
||||
self.mocked_parse.return_value = simple_mock
|
||||
|
||||
timeline = TwitterTimelineFactory(
|
||||
user__twitter_oauth_token=str(uuid4()),
|
||||
user__twitter_oauth_token_secret=str(uuid4()),
|
||||
screen_name="RobertsSpaceInd",
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertCountEqual(
|
||||
Post.objects.values_list("remote_identifier", flat=True),
|
||||
("1307054882210435074", "1307029168941461504"),
|
||||
)
|
||||
|
||||
self.assertEquals(timeline.succeeded, True)
|
||||
self.assertEquals(timeline.last_run, timezone.now())
|
||||
self.assertIsNone(timeline.error)
|
||||
|
||||
post = Post.objects.get(
|
||||
remote_identifier="1307054882210435074",
|
||||
rule__type=RuleTypeChoices.twitter_timeline,
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 20, 32, 22))
|
||||
)
|
||||
|
||||
title = truncate_text(
|
||||
Post,
|
||||
"title",
|
||||
"It's a close match-up for #SCShipShowdown today! Which Aegis ship"
|
||||
" do you think will make it to the Semi-Finals?",
|
||||
)
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(post.title, title)
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307054882210435074"
|
||||
)
|
||||
|
||||
post = Post.objects.get(
|
||||
remote_identifier="1307029168941461504",
|
||||
rule__type=RuleTypeChoices.twitter_timeline,
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 18, 50, 11))
|
||||
)
|
||||
|
||||
body = fix_text(
|
||||
"We\u2019re welcoming members of our Builds, Publishes and Platform"
|
||||
" teams on Star Citizen Live to talk about the process involved in"
|
||||
" bringing everyone\u2019s work together and getting it out into your"
|
||||
" hands. Going live on #Twitch in 10 minutes."
|
||||
" \ud83c\udfa5\ud83d\udd34 \n\nTune in:"
|
||||
" https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9"
|
||||
)
|
||||
|
||||
title = truncate_text(Post, "title", body)
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(post.title, title)
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307029168941461504"
|
||||
)
|
||||
|
||||
def test_empty_batch(self):
|
||||
self.mocked_parse.return_value = empty_mock
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
|
||||
self.assertEquals(timeline.succeeded, True)
|
||||
self.assertEquals(timeline.last_run, timezone.now())
|
||||
self.assertIsNone(timeline.error)
|
||||
|
||||
def test_not_found(self):
|
||||
self.mocked_fetch.side_effect = StreamNotFoundException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream not found")
|
||||
|
||||
def test_denied(self):
|
||||
self.mocked_fetch.side_effect = StreamDeniedException
|
||||
|
||||
timeline = TwitterTimelineFactory(
|
||||
user__twitter_oauth_token=str(uuid4()),
|
||||
user__twitter_oauth_token_secret=str(uuid4()),
|
||||
)
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream does not have sufficient permissions")
|
||||
|
||||
user = timeline.user
|
||||
|
||||
self.assertIsNone(user.twitter_oauth_token)
|
||||
self.assertIsNone(user.twitter_oauth_token_secret)
|
||||
|
||||
def test_forbidden(self):
|
||||
self.mocked_fetch.side_effect = StreamForbiddenException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream forbidden")
|
||||
|
||||
def test_timed_out(self):
|
||||
self.mocked_fetch.side_effect = StreamTimeOutException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream timed out")
|
||||
|
|
@ -154,7 +154,7 @@ class TwitterBuilder(PostBuilder):
|
|||
|
||||
|
||||
class TwitterStream(PostStream):
|
||||
rule_type = RuleTypeChoices.subreddit
|
||||
rule_type = RuleTypeChoices.twitter_timeline
|
||||
|
||||
def read(self):
|
||||
oauth = OAuth(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue