Add TwitterStream & tests
This commit is contained in:
parent
e21c875928
commit
3db3a05ca0
4 changed files with 353 additions and 3 deletions
|
|
@ -219,12 +219,10 @@ class RedditBuilder(PostBuilder):
|
|||
class RedditStream(PostStream):
|
||||
rule_type = RuleTypeChoices.subreddit
|
||||
headers = {}
|
||||
user = None
|
||||
|
||||
def __init__(self, rule):
|
||||
super().__init__(rule)
|
||||
|
||||
self.user = self.rule.user
|
||||
self.headers = {
|
||||
f"Authorization": f"bearer {self.rule.user.reddit_access_token}"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,225 @@
|
|||
# retrieved with:
|
||||
# curl -X GET -H "Authorization: Bearer <TOKEN>" "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys
|
||||
|
||||
simple_mock = [
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 20:32:22 +0000 2020",
|
||||
"display_text_range": [0, 111],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}],
|
||||
"symbols": [],
|
||||
"urls": [],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"favorite_count": 54,
|
||||
"favorited": False,
|
||||
"full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?",
|
||||
"geo": None,
|
||||
"id": 1307054882210435074,
|
||||
"id_str": "1307054882210435074",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"retweet_count": 9,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 18:50:11 +0000 2020",
|
||||
"display_text_range": [0, 271],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [211, 218], "text": "Twitch"}],
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
],
|
||||
"symbols": [],
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "twitch.tv/starcitizen",
|
||||
"expanded_url": "http://twitch.tv/starcitizen",
|
||||
"indices": [248, 271],
|
||||
"url": "https://t.co/2AdNovhpFW",
|
||||
}
|
||||
],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"extended_entities": {
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
]
|
||||
},
|
||||
"favorite_count": 90,
|
||||
"favorited": False,
|
||||
"full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9",
|
||||
"geo": None,
|
||||
"id": 1307029168941461504,
|
||||
"id_str": "1307029168941461504",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"possibly_sensitive": False,
|
||||
"retweet_count": 13,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
from json import JSONDecodeError
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from newsreader.news.collection.exceptions import (
|
||||
StreamDeniedException,
|
||||
StreamException,
|
||||
StreamForbiddenException,
|
||||
StreamNotFoundException,
|
||||
StreamParseException,
|
||||
StreamTimeOutException,
|
||||
StreamTooManyException,
|
||||
)
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.tests.twitter.stream.mocks import simple_mock
|
||||
from newsreader.news.collection.twitter import TwitterStream
|
||||
|
||||
|
||||
class TwitterStreamTestCase(TestCase):
|
||||
def setUp(self):
|
||||
self.patched_fetch = patch("newsreader.news.collection.twitter.fetch")
|
||||
self.mocked_fetch = self.patched_fetch.start()
|
||||
|
||||
def tearDown(self):
|
||||
patch.stopall()
|
||||
|
||||
def test_simple_stream(self):
|
||||
self.mocked_fetch.return_value.json.return_value = simple_mock
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
data, stream = stream.read()
|
||||
|
||||
self.assertEquals(data, simple_mock)
|
||||
self.assertEquals(stream, stream)
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_denied_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamDeniedException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamDeniedException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_not_found_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamNotFoundException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamNotFoundException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_time_out_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamTimeOutException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamTimeOutException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_forbidden_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamForbiddenException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamForbiddenException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_parse_exception(self):
|
||||
self.mocked_fetch.return_value.json.side_effect = JSONDecodeError(
|
||||
"No json found", "{}", 5
|
||||
)
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamParseException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
|
@ -150,7 +150,27 @@ class TwitterBuilder(PostBuilder):
|
|||
|
||||
|
||||
class TwitterStream(PostStream):
|
||||
pass
|
||||
rule_type = RuleTypeChoices.subreddit
|
||||
|
||||
def read(self):
|
||||
oauth = OAuth(
|
||||
settings.TWITTER_CONSUMER_ID,
|
||||
client_secret=settings.TWITTER_CONSUMER_SECRET,
|
||||
resource_owner_key=self.rule.user.twitter_oauth_token,
|
||||
resource_owner_secret=self.rule.user.twitter_oauth_token_secret,
|
||||
)
|
||||
|
||||
response = fetch(self.rule.url, auth=oauth)
|
||||
|
||||
return self.parse(response), self
|
||||
|
||||
def parse(self, response):
|
||||
try:
|
||||
return response.json()
|
||||
except JSONDecodeError as e:
|
||||
raise StreamParseException(
|
||||
response=response, message="Failed parsing json"
|
||||
) from e
|
||||
|
||||
|
||||
class TwitterClient(PostClient):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue