diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py
index db7e410..bacaabb 100644
--- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py
+++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py
@@ -204,405 +204,167 @@ image_mock = [
{
"contributors": None,
"coordinates": None,
- "created_at": "Fri Aug 07 00:01:56 +0000 2020",
- "display_text_range": [0, 278],
+ "created_at": "Fri Jun 05 22:51:46 +0000 2020",
"entities": {
"hashtags": [],
"media": [
{
- "display_url": "pic.twitter.com/aI5frhSKbI",
- "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291524942970777601/photo/1",
- "id": 1291524790956613633,
- "id_str": "1291524790956613633",
- "indices": [279, 302],
- "media_url": "http://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg",
- "media_url_https": "https://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg",
+ "display_url": "pic.twitter.com/VjEeDrL1iA",
+ "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1",
+ "id": 1269039233072689152,
+ "id_str": "1269039233072689152",
+ "indices": [2, 25],
+ "media_url": "http://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg",
+ "media_url_https": "https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg",
"sizes": {
- "large": {"h": 1080, "resize": "fit", "w": 1920},
- "medium": {"h": 675, "resize": "fit", "w": 1200},
- "small": {"h": 383, "resize": "fit", "w": 680},
+ "large": {"h": 1073, "resize": "fit", "w": 1125},
+ "medium": {"h": 1073, "resize": "fit", "w": 1125},
+ "small": {"h": 649, "resize": "fit", "w": 680},
"thumb": {"h": 150, "resize": "crop", "w": 150},
},
"type": "photo",
- "url": "https://t.co/aI5frhSKbI",
+ "url": "https://t.co/VjEeDrL1iA",
}
],
"symbols": [],
- "urls": [
- {
- "display_url": "youtu.be/rDy7tPf6CT8",
- "expanded_url": "https://youtu.be/rDy7tPf6CT8",
- "indices": [255, 278],
- "url": "https://t.co/trAcIxBMlX",
- }
- ],
+ "urls": [],
"user_mentions": [],
},
"extended_entities": {
"media": [
{
- "display_url": "pic.twitter.com/aI5frhSKbI",
- "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291524942970777601/photo/1",
- "id": 1291524790956613633,
- "id_str": "1291524790956613633",
- "indices": [279, 302],
- "media_url": "http://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg",
- "media_url_https": "https://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg",
+ "display_url": "pic.twitter.com/VjEeDrL1iA",
+ "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1",
+ "id": 1269039233072689152,
+ "id_str": "1269039233072689152",
+ "indices": [2, 25],
+ "media_url": "http://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg",
+ "media_url_https": "https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg",
"sizes": {
- "large": {"h": 1080, "resize": "fit", "w": 1920},
- "medium": {"h": 675, "resize": "fit", "w": 1200},
- "small": {"h": 383, "resize": "fit", "w": 680},
+ "large": {"h": 1073, "resize": "fit", "w": 1125},
+ "medium": {"h": 1073, "resize": "fit", "w": 1125},
+ "small": {"h": 649, "resize": "fit", "w": 680},
"thumb": {"h": 150, "resize": "crop", "w": 150},
},
"type": "photo",
- "url": "https://t.co/aI5frhSKbI",
- }
+ "url": "https://t.co/VjEeDrL1iA",
+ },
+ {
+ "display_url": "pic.twitter.com/VjEeDrL1iA",
+ "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1",
+ "id": 1269039233068527618,
+ "id_str": "1269039233068527618",
+ "indices": [2, 25],
+ "media_url": "http://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg",
+ "media_url_https": "https://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg",
+ "sizes": {
+ "large": {"h": 992, "resize": "fit", "w": 1472},
+ "medium": {"h": 809, "resize": "fit", "w": 1200},
+ "small": {"h": 458, "resize": "fit", "w": 680},
+ "thumb": {"h": 150, "resize": "crop", "w": 150},
+ },
+ "type": "photo",
+ "url": "https://t.co/VjEeDrL1iA",
+ },
]
},
- "favorite_count": 185,
+ "favorite_count": 2139,
"favorited": False,
- "full_text": "This week we strap into the cockpit and take a look at aviation experimentation in the Public Test Universe, before a Sprint Report affords looks at cargo decks, FPS weapons, day/night in the city, dynamic fire, and more. Also, penguiny hugs.\n\nWatch now: https://t.co/trAcIxBMlX https://t.co/aI5frhSKbI",
"geo": None,
- "id": 1291524942970777601,
- "id_str": "1291524942970777601",
+ "id": 1269039237166321664,
+ "id_str": "1269039237166321664",
"in_reply_to_screen_name": None,
"in_reply_to_status_id": None,
"in_reply_to_status_id_str": None,
"in_reply_to_user_id": None,
"in_reply_to_user_id_str": None,
"is_quote_status": False,
- "lang": "en",
+ "lang": "und",
"place": None,
"possibly_sensitive": False,
- "retweet_count": 39,
+ "possibly_sensitive_appealable": False,
+ "retweet_count": 427,
"retweeted": False,
- "source": 'Twitter Web App',
+ "source": 'Twitter for iPhone',
+ "text": "_ https://t.co/VjEeDrL1iA",
"truncated": False,
"user": {
"contributors_enabled": False,
- "created_at": "Wed Sep 05 00:58:11 +0000 2012",
+ "created_at": "Tue Nov 14 19:00:00 +0000 2017",
"default_profile": False,
"default_profile_image": False,
- "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
+ "description": "Grammy\u00ae Award Winning Beatmakr. https://t.co/SN23ei3EeC https://t.co/EkGRhZ1Bw9 https://t.co/eEb4NOmJLo",
"entities": {
- "description": {"urls": []},
+ "description": {
+ "urls": [
+ {
+ "display_url": "soundcloud.com/knxwledge",
+ "expanded_url": "http://soundcloud.com/knxwledge",
+ "indices": [32, 55],
+ "url": "https://t.co/SN23ei3EeC",
+ },
+ {
+ "display_url": "knxwledge.bandcamp.com",
+ "expanded_url": "http://knxwledge.bandcamp.com",
+ "indices": [56, 79],
+ "url": "https://t.co/EkGRhZ1Bw9",
+ },
+ {
+ "display_url": "twitch.tv/knxwledge",
+ "expanded_url": "http://twitch.tv/knxwledge",
+ "indices": [80, 103],
+ "url": "https://t.co/eEb4NOmJLo",
+ },
+ ]
+ },
"url": {
"urls": [
{
- "display_url": "robertsspaceindustries.com",
- "expanded_url": "http://www.robertsspaceindustries.com",
+ "display_url": "instagram.com/knxwledge/?hl=\u2026",
+ "expanded_url": "https://www.instagram.com/knxwledge/?hl=en",
"indices": [0, 23],
- "url": "https://t.co/iqO6apof3y",
+ "url": "https://t.co/UcMYfiQXLx",
}
]
},
},
- "favourites_count": 4588,
+ "favourites_count": 363,
"follow_request_sent": None,
- "followers_count": 106169,
+ "followers_count": 31194,
"following": None,
- "friends_count": 201,
+ "friends_count": 15,
"geo_enabled": False,
"has_extended_profile": False,
- "id": 803542770,
- "id_str": "803542770",
+ "id": 930510644763287552,
+ "id_str": "930510644763287552",
"is_translation_enabled": False,
"is_translator": False,
"lang": None,
- "listed_count": 890,
- "location": "Roberts Space Industries",
- "name": "Star Citizen",
+ "listed_count": 56,
+ "location": "",
+ "name": "knxwledge",
"notifications": None,
- "profile_background_color": "131516",
- "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
- "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
+ "profile_background_color": "000000",
+ "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+ "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": False,
- "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
- "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
- "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
- "profile_link_color": "0A5485",
- "profile_sidebar_border_color": "FFFFFF",
- "profile_sidebar_fill_color": "EFEFEF",
- "profile_text_color": "333333",
- "profile_use_background_image": True,
+ "profile_image_url": "http://pbs.twimg.com/profile_images/1274913160898592768/jFi4VDtJ_normal.jpg",
+ "profile_image_url_https": "https://pbs.twimg.com/profile_images/1274913160898592768/jFi4VDtJ_normal.jpg",
+ "profile_link_color": "ABB8C2",
+ "profile_sidebar_border_color": "000000",
+ "profile_sidebar_fill_color": "000000",
+ "profile_text_color": "000000",
+ "profile_use_background_image": False,
"protected": False,
- "screen_name": "RobertsSpaceInd",
- "statuses_count": 6210,
+ "screen_name": "knxwledge",
+ "statuses_count": 713,
"time_zone": None,
"translator_type": "none",
- "url": "https://t.co/iqO6apof3y",
+ "url": "https://t.co/UcMYfiQXLx",
"utc_offset": None,
- "verified": True,
+ "verified": False,
},
- },
- {
- "contributors": None,
- "coordinates": None,
- "created_at": "Thu Aug 06 00:03:04 +0000 2020",
- "display_text_range": [0, 261],
- "entities": {
- "hashtags": [{"indices": [210, 222], "text": "StarCitizen"}],
- "media": [
- {
- "display_url": "pic.twitter.com/mHekGRycKa",
- "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291162838791335936/photo/1",
- "id": 1291160447526121473,
- "id_str": "1291160447526121473",
- "indices": [262, 285],
- "media_url": "http://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg",
- "media_url_https": "https://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg",
- "sizes": {
- "large": {"h": 1078, "resize": "fit", "w": 2048},
- "medium": {"h": 632, "resize": "fit", "w": 1200},
- "small": {"h": 358, "resize": "fit", "w": 680},
- "thumb": {"h": 150, "resize": "crop", "w": 150},
- },
- "type": "photo",
- "url": "https://t.co/mHekGRycKa",
- }
- ],
- "symbols": [],
- "urls": [
- {
- "display_url": "robertsspaceindustries.com/comm-link/tran\u2026",
- "expanded_url": "https://robertsspaceindustries.com/comm-link/transmission/17712-Star-Citizen-Monthly-Report-July-2020",
- "indices": [238, 261],
- "url": "https://t.co/oVUjK6UWgC",
- }
- ],
- "user_mentions": [],
- },
- "extended_entities": {
- "media": [
- {
- "display_url": "pic.twitter.com/mHekGRycKa",
- "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291162838791335936/photo/1",
- "id": 1291160447526121473,
- "id_str": "1291160447526121473",
- "indices": [262, 285],
- "media_url": "http://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg",
- "media_url_https": "https://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg",
- "sizes": {
- "large": {"h": 1078, "resize": "fit", "w": 2048},
- "medium": {"h": 632, "resize": "fit", "w": 1200},
- "small": {"h": 358, "resize": "fit", "w": 680},
- "thumb": {"h": 150, "resize": "crop", "w": 150},
- },
- "type": "photo",
- "url": "https://t.co/mHekGRycKa",
- }
- ]
- },
- "favorite_count": 149,
- "favorited": False,
- "full_text": "We hope you\u2019re all enjoying the new additions that Alpha 3.10 brings to the \u2018verse. July\u2019s monthly report touches on a few of the final touches to the latest patch and work you can expect to see in the future. #StarCitizen\n\nRead it here: https://t.co/oVUjK6UWgC https://t.co/mHekGRycKa",
- "geo": None,
- "id": 1291162838791335936,
- "id_str": "1291162838791335936",
- "in_reply_to_screen_name": None,
- "in_reply_to_status_id": None,
- "in_reply_to_status_id_str": None,
- "in_reply_to_user_id": None,
- "in_reply_to_user_id_str": None,
- "is_quote_status": False,
- "lang": "en",
- "place": None,
- "possibly_sensitive": False,
- "retweet_count": 28,
- "retweeted": False,
- "source": 'Twitter Web App',
- "truncated": False,
- "user": {
- "contributors_enabled": False,
- "created_at": "Wed Sep 05 00:58:11 +0000 2012",
- "default_profile": False,
- "default_profile_image": False,
- "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
- "entities": {
- "description": {"urls": []},
- "url": {
- "urls": [
- {
- "display_url": "robertsspaceindustries.com",
- "expanded_url": "http://www.robertsspaceindustries.com",
- "indices": [0, 23],
- "url": "https://t.co/iqO6apof3y",
- }
- ]
- },
- },
- "favourites_count": 4588,
- "follow_request_sent": None,
- "followers_count": 106169,
- "following": None,
- "friends_count": 201,
- "geo_enabled": False,
- "has_extended_profile": False,
- "id": 803542770,
- "id_str": "803542770",
- "is_translation_enabled": False,
- "is_translator": False,
- "lang": None,
- "listed_count": 890,
- "location": "Roberts Space Industries",
- "name": "Star Citizen",
- "notifications": None,
- "profile_background_color": "131516",
- "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
- "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
- "profile_background_tile": False,
- "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
- "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
- "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
- "profile_link_color": "0A5485",
- "profile_sidebar_border_color": "FFFFFF",
- "profile_sidebar_fill_color": "EFEFEF",
- "profile_text_color": "333333",
- "profile_use_background_image": True,
- "protected": False,
- "screen_name": "RobertsSpaceInd",
- "statuses_count": 6210,
- "time_zone": None,
- "translator_type": "none",
- "url": "https://t.co/iqO6apof3y",
- "utc_offset": None,
- "verified": True,
- },
- },
- {
- "contributors": None,
- "coordinates": None,
- "created_at": "Wed Aug 05 19:31:23 +0000 2020",
- "display_text_range": [0, 218],
- "entities": {
- "hashtags": [],
- "media": [
- {
- "display_url": "pic.twitter.com/ESB5UBwhmO",
- "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291094468473393152/photo/1",
- "id": 1291090033080856576,
- "id_str": "1291090033080856576",
- "indices": [219, 242],
- "media_url": "http://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg",
- "media_url_https": "https://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg",
- "sizes": {
- "large": {"h": 635, "resize": "fit", "w": 1680},
- "medium": {"h": 454, "resize": "fit", "w": 1200},
- "small": {"h": 257, "resize": "fit", "w": 680},
- "thumb": {"h": 150, "resize": "crop", "w": 150},
- },
- "type": "photo",
- "url": "https://t.co/ESB5UBwhmO",
- }
- ],
- "symbols": [],
- "urls": [
- {
- "display_url": "robertsspaceindustries.com/spectrum/commu\u2026",
- "expanded_url": "https://robertsspaceindustries.com/spectrum/community/SC/forum/3/thread/where-in-the-verse-is-pico-screenshot-contest",
- "indices": [195, 218],
- "url": "https://t.co/jFU7YLniDR",
- }
- ],
- "user_mentions": [],
- },
- "extended_entities": {
- "media": [
- {
- "display_url": "pic.twitter.com/ESB5UBwhmO",
- "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291094468473393152/photo/1",
- "id": 1291090033080856576,
- "id_str": "1291090033080856576",
- "indices": [219, 242],
- "media_url": "http://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg",
- "media_url_https": "https://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg",
- "sizes": {
- "large": {"h": 635, "resize": "fit", "w": 1680},
- "medium": {"h": 454, "resize": "fit", "w": 1200},
- "small": {"h": 257, "resize": "fit", "w": 680},
- "thumb": {"h": 150, "resize": "crop", "w": 150},
- },
- "type": "photo",
- "url": "https://t.co/ESB5UBwhmO",
- }
- ]
- },
- "favorite_count": 306,
- "favorited": False,
- "full_text": "Alpha 3.10 is live and our fuzzy friend Pico wants in on the action. \ud83d\udc27 We want to see what you and Pico are getting up to in the 'verse with our \"Where in the 'Verse is Pico?\" contest!\n\nDetails: https://t.co/jFU7YLniDR https://t.co/ESB5UBwhmO",
- "geo": None,
- "id": 1291094468473393152,
- "id_str": "1291094468473393152",
- "in_reply_to_screen_name": None,
- "in_reply_to_status_id": None,
- "in_reply_to_status_id_str": None,
- "in_reply_to_user_id": None,
- "in_reply_to_user_id_str": None,
- "is_quote_status": False,
- "lang": "en",
- "place": None,
- "possibly_sensitive": False,
- "retweet_count": 65,
- "retweeted": False,
- "source": 'Twitter Web App',
- "truncated": False,
- "user": {
- "contributors_enabled": False,
- "created_at": "Wed Sep 05 00:58:11 +0000 2012",
- "default_profile": False,
- "default_profile_image": False,
- "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
- "entities": {
- "description": {"urls": []},
- "url": {
- "urls": [
- {
- "display_url": "robertsspaceindustries.com",
- "expanded_url": "http://www.robertsspaceindustries.com",
- "indices": [0, 23],
- "url": "https://t.co/iqO6apof3y",
- }
- ]
- },
- },
- "favourites_count": 4588,
- "follow_request_sent": None,
- "followers_count": 106169,
- "following": None,
- "friends_count": 201,
- "geo_enabled": False,
- "has_extended_profile": False,
- "id": 803542770,
- "id_str": "803542770",
- "is_translation_enabled": False,
- "is_translator": False,
- "lang": None,
- "listed_count": 890,
- "location": "Roberts Space Industries",
- "name": "Star Citizen",
- "notifications": None,
- "profile_background_color": "131516",
- "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
- "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
- "profile_background_tile": False,
- "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
- "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
- "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
- "profile_link_color": "0A5485",
- "profile_sidebar_border_color": "FFFFFF",
- "profile_sidebar_fill_color": "EFEFEF",
- "profile_text_color": "333333",
- "profile_use_background_image": True,
- "protected": False,
- "screen_name": "RobertsSpaceInd",
- "statuses_count": 6210,
- "time_zone": None,
- "translator_type": "none",
- "url": "https://t.co/iqO6apof3y",
- "utc_offset": None,
- "verified": True,
- },
- },
+ }
]
video_mock = [
diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py
index 2775841..a33fc84 100644
--- a/src/newsreader/news/collection/twitter.py
+++ b/src/newsreader/news/collection/twitter.py
@@ -11,6 +11,7 @@ from newsreader.news.core.models import Post
TWITTER_URL = "https://twitter.com"
+TWITTER_API_URL = "https://api.twitter.com/1.1"
class TwitterScheduler: