From 13a87d5ecb8cef95e85015e009311c9be9f0851b Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Thu, 6 Aug 2020 22:43:53 +0200 Subject: [PATCH 01/69] Initial commit --- src/newsreader/news/collection/choices.py | 1 + .../news/collection/tests/twitter/__init__.py | 0 .../tests/twitter/builder/__init__.py | 0 .../collection/tests/twitter/builder/mocks.py | 3546 +++++++++++++++++ .../collection/tests/twitter/builder/tests.py | 0 .../tests/twitter/client/__init__.py | 0 .../collection/tests/twitter/client/mocks.py | 0 .../collection/tests/twitter/client/tests.py | 0 .../tests/twitter/collector/__init__.py | 0 .../tests/twitter/collector/mocks.py | 0 .../tests/twitter/collector/tests.py | 0 .../tests/twitter/stream/__init__.py | 0 .../collection/tests/twitter/stream/mocks.py | 0 .../collection/tests/twitter/stream/tests.py | 0 src/newsreader/news/collection/twitter.py | 32 + 15 files changed, 3579 insertions(+) create mode 100644 src/newsreader/news/collection/tests/twitter/__init__.py create mode 100644 src/newsreader/news/collection/tests/twitter/builder/__init__.py create mode 100644 src/newsreader/news/collection/tests/twitter/builder/mocks.py create mode 100644 src/newsreader/news/collection/tests/twitter/builder/tests.py create mode 100644 src/newsreader/news/collection/tests/twitter/client/__init__.py create mode 100644 src/newsreader/news/collection/tests/twitter/client/mocks.py create mode 100644 src/newsreader/news/collection/tests/twitter/client/tests.py create mode 100644 src/newsreader/news/collection/tests/twitter/collector/__init__.py create mode 100644 src/newsreader/news/collection/tests/twitter/collector/mocks.py create mode 100644 src/newsreader/news/collection/tests/twitter/collector/tests.py create mode 100644 src/newsreader/news/collection/tests/twitter/stream/__init__.py create mode 100644 src/newsreader/news/collection/tests/twitter/stream/mocks.py create mode 100644 src/newsreader/news/collection/tests/twitter/stream/tests.py create mode 100644 src/newsreader/news/collection/twitter.py diff --git a/src/newsreader/news/collection/choices.py b/src/newsreader/news/collection/choices.py index 65f7ef5..47e304e 100644 --- a/src/newsreader/news/collection/choices.py +++ b/src/newsreader/news/collection/choices.py @@ -5,3 +5,4 @@ from django.utils.translation import gettext as _ class RuleTypeChoices(TextChoices): feed = "feed", _("Feed") subreddit = "subreddit", _("Subreddit") + twitter = "twitter", _("Twitter") diff --git a/src/newsreader/news/collection/tests/twitter/__init__.py b/src/newsreader/news/collection/tests/twitter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/builder/__init__.py b/src/newsreader/news/collection/tests/twitter/builder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py new file mode 100644 index 0000000..22a2d94 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -0,0 +1,3546 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=twitterapi" | python3 -m json.tool --sort-keys + +example_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Apr 29 17:03:24 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1255543219087044608, + "id_str": "1255543219087044608", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 329, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Apr 29 17:01:38 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1255542774432063488", + "indices": [121, 144], + "url": "https://t.co/X5ZVcb6m1j", + } + ], + "user_mentions": [], + }, + "favorite_count": 752, + "favorited": False, + "geo": None, + "id": 1255542774432063488, + "id_str": "1255542774432063488", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 329, + "retweeted": False, + "source": 'Twitter Web App', + "text": "During these unprecedented times, what’s happening on Twitter can help the world better understand & respond to the… https://t.co/X5ZVcb6m1j", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: During these unprecedented times, what’s happening on Twitter can help the world better understand & respond to the pandemi…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Mon Mar 23 22:14:35 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1242213180060758016, + "id_str": "1242213180060758016", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 46, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Mon Mar 23 22:01:13 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1242209814706438144", + "indices": [116, 139], + "url": "https://t.co/Oo1t07UH4Z", + } + ], + "user_mentions": [], + }, + "favorite_count": 171, + "favorited": False, + "geo": None, + "id": 1242209814706438144, + "id_str": "1242209814706438144", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 46, + "retweeted": False, + "source": 'Twitter Web App', + "text": "As we work to keep our employees safe during COVID-19, you are likely to experience longer than usual review times… https://t.co/Oo1t07UH4Z", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: As we work to keep our employees safe during COVID-19, you are likely to experience longer than usual review times for deve…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Mon Mar 23 16:28:09 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1242125997081673728, + "id_str": "1242125997081673728", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "en", + "place": None, + "quoted_status_id": 1204497856679202816, + "quoted_status_id_str": "1204497856679202816", + "retweet_count": 41, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Mon Mar 23 16:26:08 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1242125486844604425", + "indices": [117, 140], + "url": "https://t.co/ViHyvQ4Y8S", + } + ], + "user_mentions": [], + }, + "favorite_count": 120, + "favorited": False, + "geo": None, + "id": 1242125486844604425, + "id_str": "1242125486844604425", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "quoted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Dec 10 20:27:22 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1204497856679202816", + "indices": [117, 140], + "url": "https://t.co/ZlJUjmHIBe", + } + ], + "user_mentions": [], + }, + "favorite_count": 255, + "favorited": False, + "geo": None, + "id": 1204497856679202816, + "id_str": "1204497856679202816", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 95, + "retweeted": False, + "source": 'Twitter Web App', + "text": "You may have seen the recent announcement about following Topics on Twitter. Today, we’re excited to provide API su… https://t.co/ZlJUjmHIBe", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "quoted_status_id": 1204497856679202816, + "quoted_status_id_str": "1204497856679202816", + "retweet_count": 41, + "retweeted": False, + "source": 'Twitter Web App', + "text": "A few months ago, we added Tweet annotations to the Labs’ streaming endpoints. These annotations help uncover detai… https://t.co/ViHyvQ4Y8S", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: A few months ago, we added Tweet annotations to the Labs’ streaming endpoints. These annotations help uncover details about…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Tue Mar 10 17:57:58 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1237437557337513984, + "id_str": "1237437557337513984", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 27, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Mar 10 17:47:53 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1237435017883762689", + "indices": [117, 140], + "url": "https://t.co/NpF4h9DaSq", + } + ], + "user_mentions": [], + }, + "favorite_count": 97, + "favorited": False, + "geo": None, + "id": 1237435017883762689, + "id_str": "1237435017883762689", + "in_reply_to_screen_name": "TwitterDev", + "in_reply_to_status_id": 1237435016134656006, + "in_reply_to_status_id_str": "1237435016134656006", + "in_reply_to_user_id": 2244994945, + "in_reply_to_user_id_str": "2244994945", + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 27, + "retweeted": False, + "source": 'Twitter Web App', + "text": "We ❤️ the incredible research people do using Twitter data to study topics like spam, abuse, and other areas relate… https://t.co/NpF4h9DaSq", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: We ❤️ the incredible research people do using Twitter data to study topics like spam, abuse, and other areas related to the…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Feb 26 17:33:41 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1232720402700521474, + "id_str": "1232720402700521474", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 148, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Feb 26 17:32:51 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1232720193182412800", + "indices": [117, 140], + "url": "https://t.co/aN8kan0Lsw", + } + ], + "user_mentions": [], + }, + "favorite_count": 386, + "favorited": False, + "geo": None, + "id": 1232720193182412800, + "id_str": "1232720193182412800", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 148, + "retweeted": False, + "source": 'Twitter Web App', + "text": "In November, we gave people the ability to hide replies to their Tweets. Starting today, we’re opening this feature… https://t.co/aN8kan0Lsw", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: In November, we gave people the ability to hide replies to their Tweets. Starting today, we’re opening this feature up to d…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Mon Jan 06 20:23:49 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1214281438092238855, + "id_str": "1214281438092238855", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 59, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Mon Jan 06 20:22:05 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1214281000932593667", + "indices": [117, 140], + "url": "https://t.co/XaqD1JJ5kF", + } + ], + "user_mentions": [], + }, + "favorite_count": 166, + "favorited": False, + "geo": None, + "id": 1214281000932593667, + "id_str": "1214281000932593667", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 59, + "retweeted": False, + "source": 'Twitter Web App', + "text": "Hello… is it me you’re searching for? 🔎\n\nSearch the conversation as it unfolds with this new addition to Labs. We'r… https://t.co/XaqD1JJ5kF", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: Hello… is it me you’re searching for? 🔎\n\nSearch the conversation as it unfolds with this new addition to Labs. We're making…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jan 03 17:21:34 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1213148410145992704, + "id_str": "1213148410145992704", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 218, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jan 03 17:17:23 +0000 2020", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1213147357551816704", + "indices": [117, 140], + "url": "https://t.co/WhV7rP54GM", + } + ], + "user_mentions": [], + }, + "favorite_count": 450, + "favorited": False, + "geo": None, + "id": 1213147357551816704, + "id_str": "1213147357551816704", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 218, + "retweeted": False, + "source": 'Twitter Web App', + "text": "Today, we’re sharing a few small improvements to make it easier for academic researchers to get started with the Tw… https://t.co/WhV7rP54GM", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: Today, we’re sharing a few small improvements to make it easier for academic researchers to get started with the Twitter AP…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Tue Dec 10 20:34:57 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1204499768459661312, + "id_str": "1204499768459661312", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 95, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Dec 10 20:27:22 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1204497856679202816", + "indices": [117, 140], + "url": "https://t.co/ZlJUjmHIBe", + } + ], + "user_mentions": [], + }, + "favorite_count": 255, + "favorited": False, + "geo": None, + "id": 1204497856679202816, + "id_str": "1204497856679202816", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 95, + "retweeted": False, + "source": 'Twitter Web App', + "text": "You may have seen the recent announcement about following Topics on Twitter. Today, we’re excited to provide API su… https://t.co/ZlJUjmHIBe", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: You may have seen the recent announcement about following Topics on Twitter. Today, we’re excited to provide API support fo…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Tue Oct 29 19:39:40 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1189265562821640193, + "id_str": "1189265562821640193", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 46, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Oct 29 19:37:15 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1189264953368338432", + "indices": [110, 133], + "url": "https://t.co/Y5QMh8rNoh", + } + ], + "user_mentions": [], + }, + "favorite_count": 145, + "favorited": False, + "geo": None, + "id": 1189264953368338432, + "id_str": "1189264953368338432", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 46, + "retweeted": False, + "source": 'Twitter Web App', + "text": "Study a sample of timely, relevant Tweets as they happen, with the newest release in Twitter Developer Labs.… https://t.co/Y5QMh8rNoh", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: Study a sample of timely, relevant Tweets as they happen, with the newest release in Twitter Developer Labs. https://t.co/m…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Sep 18 16:41:39 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1174362863118372867, + "id_str": "1174362863118372867", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 106, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Sep 18 16:36:10 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1174361480734466048", + "indices": [117, 140], + "url": "https://t.co/RElmBW5XxM", + } + ], + "user_mentions": [], + }, + "favorite_count": 234, + "favorited": False, + "geo": None, + "id": 1174361480734466048, + "id_str": "1174361480734466048", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 106, + "retweeted": False, + "source": 'Twitter Web App', + "text": "Today in Twitter Developer Labs we’re releasing a new way to filter Tweets in real-time. This is one of the most po… https://t.co/RElmBW5XxM", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: Today in Twitter Developer Labs we’re releasing a new way to filter Tweets in real-time. This is one of the most popular fe…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Tue Aug 27 17:30:39 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1166402661282746368, + "id_str": "1166402661282746368", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 59, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Aug 27 17:25:06 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1166401263170281472", + "indices": [117, 140], + "url": "https://t.co/a8PaA1wg5A", + } + ], + "user_mentions": [], + }, + "favorite_count": 174, + "favorited": False, + "geo": None, + "id": 1166401263170281472, + "id_str": "1166401263170281472", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 59, + "retweeted": False, + "source": 'Twitter Web App', + "text": "Our latest Twitter Developer Labs release helps you quickly assess the impact of your Tweets. Today, we’re releasin… https://t.co/a8PaA1wg5A", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: Our latest Twitter Developer Labs release helps you quickly assess the impact of your Tweets. Today, we’re releasing \n\n✨a n…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Aug 16 17:25:53 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1162415194749882368, + "id_str": "1162415194749882368", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 31, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Fri Aug 16 17:07:22 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1162410535121387525", + "indices": [117, 140], + "url": "https://t.co/VemT1Licd9", + } + ], + "user_mentions": [], + }, + "favorite_count": 102, + "favorited": False, + "geo": None, + "id": 1162410535121387525, + "id_str": "1162410535121387525", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 31, + "retweeted": False, + "source": 'Twitter Web App', + "text": "As of today, we’re simplifying permissions for third-party apps. Most developers won't be impacted, but if your app… https://t.co/VemT1Licd9", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter for iPhone', + "text": "RT @TwitterDev: As of today, we’re simplifying permissions for third-party apps. Most developers won't be impacted, but if your app uses th…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Thu Aug 15 16:24:23 +0000 2019", + "entities": { + "hashtags": [{"indices": [16, 26], "text": "iterating"}], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1162037328442875904, + "id_str": "1162037328442875904", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 26, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Thu Aug 15 16:20:05 +0000 2019", + "entities": { + "hashtags": [{"indices": [0, 10], "text": "iterating"}], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1162036247314833408", + "indices": [117, 140], + "url": "https://t.co/rRlVHaBTs2", + } + ], + "user_mentions": [], + }, + "favorite_count": 102, + "favorited": False, + "geo": None, + "id": 1162036247314833408, + "id_str": "1162036247314833408", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 26, + "retweeted": False, + "source": 'Twitter Web App', + "text": "#iterating: We recently released a long-requested feature in Twitter Developer Labs: \n\n📌Developer’s can now request… https://t.co/rRlVHaBTs2", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: #iterating: We recently released a long-requested feature in Twitter Developer Labs: \n\n📌Developer’s can now request a user’…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 31 19:23:15 +0000 2019", + "entities": { + "hashtags": [{"indices": [16, 26], "text": "iterating"}], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1156646525121916928, + "id_str": "1156646525121916928", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 41, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 31 19:20:25 +0000 2019", + "entities": { + "hashtags": [{"indices": [0, 10], "text": "iterating"}], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1156645810156650496", + "indices": [117, 140], + "url": "https://t.co/VVp7rv6FIM", + } + ], + "user_mentions": [], + }, + "favorite_count": 111, + "favorited": False, + "geo": None, + "id": 1156645810156650496, + "id_str": "1156645810156650496", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 41, + "retweeted": False, + "source": 'Twitter Web App', + "text": "#iterating: Today we are releasing an update to Twitter Developer Labs, with a few new features we think you’ll fin… https://t.co/VVp7rv6FIM", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'Twitter Web App', + "text": "RT @TwitterDev: #iterating: Today we are releasing an update to Twitter Developer Labs, with a few new features we think you’ll find useful…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 24 15:56:09 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1154057692723519494", + "indices": [117, 140], + "url": "https://t.co/8YgCwYoE3q", + } + ], + "user_mentions": [], + }, + "favorite_count": 128, + "favorited": False, + "geo": None, + "id": 1154057692723519494, + "id_str": "1154057692723519494", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "quoted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Jun 11 22:13:27 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1138569964032385025", + "indices": [117, 140], + "url": "https://t.co/qMtoumuG1e", + } + ], + "user_mentions": [], + }, + "favorite_count": 121, + "favorited": False, + "geo": None, + "id": 1138569964032385025, + "id_str": "1138569964032385025", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 91, + "retweeted": False, + "source": 'Twitter Web Client', + "text": "Starting July 15, 2019, all connections to the Twitter API (and all other Twitter domains) will require TLS 1.2. Re… https://t.co/qMtoumuG1e", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + "quoted_status_id": 1138569964032385025, + "quoted_status_id_str": "1138569964032385025", + "retweet_count": 108, + "retweeted": False, + "source": 'Twitter Web App', + "text": "TLS 1.2 reminder: this change will be enacted as of tomorrow, July 25, 2019. Please reference our developer forum p… https://t.co/8YgCwYoE3q", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 17 15:54:45 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 2244994945, + "id_str": "2244994945", + "indices": [3, 14], + "name": "Twitter Dev", + "screen_name": "TwitterDev", + } + ], + }, + "favorite_count": 0, + "favorited": False, + "geo": None, + "id": 1151520624315174912, + "id_str": "1151520624315174912", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 421, + "retweeted": False, + "retweeted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 17 15:53:43 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1151520361529430016", + "indices": [117, 140], + "url": "https://t.co/dG3PmGWAJ4", + } + ], + "user_mentions": [], + }, + "favorite_count": 675, + "favorited": False, + "geo": None, + "id": 1151520361529430016, + "id_str": "1151520361529430016", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 421, + "retweeted": False, + "source": 'Twitter Web App', + "text": "Academic research is some of the most impactful work that happens with the Twitter API. As we plan for the future o… https://t.co/dG3PmGWAJ4", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "source": 'TweetDeck', + "text": "RT @TwitterDev: Academic research is some of the most impactful work that happens with the Twitter API. As we plan for the future of our de…", + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Mon Jun 24 17:50:46 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1143214899109277697", + "indices": [117, 140], + "url": "https://t.co/w8WoepBjeU", + } + ], + "user_mentions": [], + }, + "favorite_count": 85, + "favorited": False, + "geo": None, + "id": 1143214899109277697, + "id_str": "1143214899109277697", + "in_reply_to_screen_name": "TwitterAPI", + "in_reply_to_status_id": 1141392777600806912, + "in_reply_to_status_id_str": "1141392777600806912", + "in_reply_to_user_id": 6253282, + "in_reply_to_user_id_str": "6253282", + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 44, + "retweeted": False, + "source": 'Twitter Web Client', + "text": "We’ve spoken with all developers who’ve contacted us to discuss these new rate limits and elevations, and as of tod… https://t.co/w8WoepBjeU", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jun 19 17:10:18 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1141392777600806912", + "indices": [117, 140], + "url": "https://t.co/ymDvv7r8lB", + } + ], + "user_mentions": [], + }, + "favorite_count": 90, + "favorited": False, + "geo": None, + "id": 1141392777600806912, + "id_str": "1141392777600806912", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "quoted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jun 19 17:02:25 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1141390793657266176", + "indices": [117, 140], + "url": "https://t.co/kFzCKybdkD", + } + ], + "user_mentions": [], + }, + "favorite_count": 46, + "favorited": False, + "geo": None, + "id": 1141390793657266176, + "id_str": "1141390793657266176", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "quoted_status_id": 1108050885639168000, + "quoted_status_id_str": "1108050885639168000", + "retweet_count": 19, + "retweeted": False, + "source": 'Twitter Web Client', + "text": "❗️Today, user and mentions timeline request limits go into effect. If you want to learn more about this change, tak… https://t.co/kFzCKybdkD", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "default_profile": False, + "default_profile_image": False, + "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com/en/community", + "expanded_url": "https://developer.twitter.com/en/community", + "indices": [0, 23], + "url": "https://t.co/3ZX3TNiZCY", + } + ] + }, + }, + "favourites_count": 2165, + "follow_request_sent": None, + "followers_count": 509354, + "following": None, + "friends_count": 2000, + "geo_enabled": True, + "has_extended_profile": True, + "id": 2244994945, + "id_str": "2244994945", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 1589, + "location": "127.0.0.1", + "name": "Twitter Dev", + "notifications": None, + "profile_background_color": "FFFFFF", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", + "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": False, + "protected": False, + "screen_name": "TwitterDev", + "statuses_count": 3576, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/3ZX3TNiZCY", + "utc_offset": None, + "verified": True, + }, + }, + "quoted_status_id": 1141390793657266176, + "quoted_status_id_str": "1141390793657266176", + "retweet_count": 70, + "retweeted": False, + "source": 'Twitter Web Client', + "text": "Request limit change: today, we're implementing a change to two commonly used Twitter standard API endpoints - user… https://t.co/ymDvv7r8lB", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jun 12 17:36:26 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1138862637394137093", + "indices": [116, 139], + "url": "https://t.co/JAUtpAZotb", + } + ], + "user_mentions": [], + }, + "favorite_count": 62, + "favorited": False, + "geo": None, + "id": 1138862637394137093, + "id_str": "1138862637394137093", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 40, + "retweeted": False, + "source": 'Twitter Web Client', + "text": "Reminder: only 1⃣ week until the rate limit change to user and mentions timeline endpoints will go into effect. If… https://t.co/JAUtpAZotb", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Tue Jun 11 22:13:27 +0000 2019", + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "twitter.com/i/web/status/1…", + "expanded_url": "https://twitter.com/i/web/status/1138569964032385025", + "indices": [117, 140], + "url": "https://t.co/qMtoumuG1e", + } + ], + "user_mentions": [], + }, + "favorite_count": 121, + "favorited": False, + "geo": None, + "id": 1138569964032385025, + "id_str": "1138569964032385025", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 91, + "retweeted": False, + "source": 'Twitter Web Client', + "text": "Starting July 15, 2019, all connections to the Twitter API (and all other Twitter domains) will require TLS 1.2. Re… https://t.co/qMtoumuG1e", + "truncated": True, + "user": { + "contributors_enabled": False, + "created_at": "Wed May 23 06:01:13 +0000 2007", + "default_profile": False, + "default_profile_image": False, + "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "developer.twitter.com", + "expanded_url": "https://developer.twitter.com", + "indices": [0, 23], + "url": "https://t.co/8IkCzCDr19", + } + ] + }, + }, + "favourites_count": 3, + "follow_request_sent": None, + "followers_count": 6052538, + "following": None, + "friends_count": 32, + "geo_enabled": False, + "has_extended_profile": True, + "id": 6253282, + "id_str": "6253282", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 12307, + "location": "", + "name": "Twitter API", + "notifications": None, + "profile_background_color": "C0DEED", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": True, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", + "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", + "profile_link_color": "0084B4", + "profile_sidebar_border_color": "C0DEED", + "profile_sidebar_fill_color": "DDEEF6", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "TwitterAPI", + "statuses_count": 3679, + "time_zone": None, + "translator_type": "regular", + "url": "https://t.co/8IkCzCDr19", + "utc_offset": None, + "verified": True, + }, + }, +] diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/client/__init__.py b/src/newsreader/news/collection/tests/twitter/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/client/mocks.py b/src/newsreader/news/collection/tests/twitter/client/mocks.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/client/tests.py b/src/newsreader/news/collection/tests/twitter/client/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/collector/__init__.py b/src/newsreader/news/collection/tests/twitter/collector/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/collector/mocks.py b/src/newsreader/news/collection/tests/twitter/collector/mocks.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/collector/tests.py b/src/newsreader/news/collection/tests/twitter/collector/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/stream/__init__.py b/src/newsreader/news/collection/tests/twitter/stream/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/stream/mocks.py b/src/newsreader/news/collection/tests/twitter/stream/mocks.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/tests/twitter/stream/tests.py b/src/newsreader/news/collection/tests/twitter/stream/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py new file mode 100644 index 0000000..15a2ccd --- /dev/null +++ b/src/newsreader/news/collection/twitter.py @@ -0,0 +1,32 @@ +from newsreader.news.collection.base import Builder, Client, Collector, Stream + + +class TwitterScheduler: + pass + + +class TwitterBuilder(Builder): + def __enter__(self): + _, stream = self.stream + + self.instances = [] + self.existing_posts = { + post.remote_identifier: post + for post in Post.objects.filter( + rule=stream.rule, rule__type=RuleTypeChoices.twitter + ) + } + + return super().__enter__() + + +class TwitterStream(Stream): + pass + + +class TwitterClient(Client): + pass + + +class TwitterCollector(Collector): + pass From fec9af5626c2b280e51ebd3d130a326f6078ef12 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 7 Aug 2020 16:43:26 +0200 Subject: [PATCH 02/69] Add mocks for different kinds of media/tweets --- .../collection/tests/twitter/builder/mocks.py | 4451 ++++++----------- .../collection/tests/twitter/builder/tests.py | 62 + 2 files changed, 1591 insertions(+), 2922 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index 22a2d94..db7e410 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -1,30 +1,270 @@ # retrieved with: -# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=twitterapi" | python3 -m json.tool --sort-keys +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=twitterapi&tweet_mode=extended" | python3 -m json.tool --sort-keys -example_mock = [ +simple_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Wed Apr 29 17:03:24 +0000 2020", + "created_at": "Fri Aug 07 00:17:05 +0000 2020", + "display_text_range": [11, 59], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "youtu.be/rDy7tPf6CT8", + "expanded_url": "https://youtu.be/rDy7tPf6CT8", + "indices": [36, 59], + "url": "https://t.co/trAcIxBMlX", + } + ], + "user_mentions": [ + { + "id": 975844884606275587, + "id_str": "975844884606275587", + "indices": [0, 10], + "name": "ArieNeo", + "screen_name": "ArieNeoSC", + } + ], + }, + "favorite_count": 19, + "favorited": False, + "full_text": "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX", + "geo": None, + "id": 1291528756373286914, + "id_str": "1291528756373286914", + "in_reply_to_screen_name": "ArieNeoSC", + "in_reply_to_status_id": 1291507356313038850, + "in_reply_to_status_id_str": "1291507356313038850", + "in_reply_to_user_id": 975844884606275587, + "in_reply_to_user_id_str": "975844884606275587", + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 5, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Jul 29 19:01:47 +0000 2020", + "display_text_range": [10, 98], "entities": { "hashtags": [], "symbols": [], "urls": [], "user_mentions": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "id": 435221600, + "id_str": "435221600", + "indices": [0, 9], + "name": "Christopher Blough", + "screen_name": "RelicCcb", } ], }, - "favorite_count": 0, + "favorite_count": 1, "favorited": False, + "full_text": "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing.", "geo": None, - "id": 1255543219087044608, - "id_str": "1255543219087044608", + "id": 1288550304095416320, + "id_str": "1288550304095416320", + "in_reply_to_screen_name": "RelicCcb", + "in_reply_to_status_id": 1288475147951898625, + "in_reply_to_status_id_str": "1288475147951898625", + "in_reply_to_user_id": 435221600, + "in_reply_to_user_id_str": "435221600", + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 0, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +image_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Aug 07 00:01:56 +0000 2020", + "display_text_range": [0, 278], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/aI5frhSKbI", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291524942970777601/photo/1", + "id": 1291524790956613633, + "id_str": "1291524790956613633", + "indices": [279, 302], + "media_url": "http://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", + "media_url_https": "https://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", + "sizes": { + "large": {"h": 1080, "resize": "fit", "w": 1920}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/aI5frhSKbI", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "youtu.be/rDy7tPf6CT8", + "expanded_url": "https://youtu.be/rDy7tPf6CT8", + "indices": [255, 278], + "url": "https://t.co/trAcIxBMlX", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/aI5frhSKbI", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291524942970777601/photo/1", + "id": 1291524790956613633, + "id_str": "1291524790956613633", + "indices": [279, 302], + "media_url": "http://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", + "media_url_https": "https://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", + "sizes": { + "large": {"h": 1080, "resize": "fit", "w": 1920}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/aI5frhSKbI", + } + ] + }, + "favorite_count": 185, + "favorited": False, + "full_text": "This week we strap into the cockpit and take a look at aviation experimentation in the Public Test Universe, before a Sprint Report affords looks at cargo decks, FPS weapons, day/night in the city, dynamic fire, and more. Also, penguiny hugs.\n\nWatch now: https://t.co/trAcIxBMlX https://t.co/aI5frhSKbI", + "geo": None, + "id": 1291524942970777601, + "id_str": "1291524942970777601", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -33,157 +273,64 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 329, + "possibly_sensitive": False, + "retweet_count": 39, "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Wed Apr 29 17:01:38 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1255542774432063488", - "indices": [121, 144], - "url": "https://t.co/X5ZVcb6m1j", - } - ], - "user_mentions": [], - }, - "favorite_count": 752, - "favorited": False, - "geo": None, - "id": 1255542774432063488, - "id_str": "1255542774432063488", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 329, - "retweeted": False, - "source": 'Twitter Web App', - "text": "During these unprecedented times, what’s happening on Twitter can help the world better understand & respond to the… https://t.co/X5ZVcb6m1j", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, "source": 'Twitter Web App', - "text": "RT @TwitterDev: During these unprecedented times, what’s happening on Twitter can help the world better understand & respond to the pandemi…", "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, @@ -191,26 +338,67 @@ example_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Mon Mar 23 22:14:35 +0000 2020", + "created_at": "Thu Aug 06 00:03:04 +0000 2020", + "display_text_range": [0, 261], "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ + "hashtags": [{"indices": [210, 222], "text": "StarCitizen"}], + "media": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "display_url": "pic.twitter.com/mHekGRycKa", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291162838791335936/photo/1", + "id": 1291160447526121473, + "id_str": "1291160447526121473", + "indices": [262, 285], + "media_url": "http://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", + "media_url_https": "https://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", + "sizes": { + "large": {"h": 1078, "resize": "fit", "w": 2048}, + "medium": {"h": 632, "resize": "fit", "w": 1200}, + "small": {"h": 358, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/mHekGRycKa", } ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/comm-link/tran\u2026", + "expanded_url": "https://robertsspaceindustries.com/comm-link/transmission/17712-Star-Citizen-Monthly-Report-July-2020", + "indices": [238, 261], + "url": "https://t.co/oVUjK6UWgC", + } + ], + "user_mentions": [], }, - "favorite_count": 0, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/mHekGRycKa", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291162838791335936/photo/1", + "id": 1291160447526121473, + "id_str": "1291160447526121473", + "indices": [262, 285], + "media_url": "http://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", + "media_url_https": "https://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", + "sizes": { + "large": {"h": 1078, "resize": "fit", "w": 2048}, + "medium": {"h": 632, "resize": "fit", "w": 1200}, + "small": {"h": 358, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/mHekGRycKa", + } + ] + }, + "favorite_count": 149, "favorited": False, + "full_text": "We hope you\u2019re all enjoying the new additions that Alpha 3.10 brings to the \u2018verse. July\u2019s monthly report touches on a few of the final touches to the latest patch and work you can expect to see in the future. #StarCitizen\n\nRead it here: https://t.co/oVUjK6UWgC https://t.co/mHekGRycKa", "geo": None, - "id": 1242213180060758016, - "id_str": "1242213180060758016", + "id": 1291162838791335936, + "id_str": "1291162838791335936", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -219,157 +407,64 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 46, + "possibly_sensitive": False, + "retweet_count": 28, "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Mon Mar 23 22:01:13 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1242209814706438144", - "indices": [116, 139], - "url": "https://t.co/Oo1t07UH4Z", - } - ], - "user_mentions": [], - }, - "favorite_count": 171, - "favorited": False, - "geo": None, - "id": 1242209814706438144, - "id_str": "1242209814706438144", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 46, - "retweeted": False, - "source": 'Twitter Web App', - "text": "As we work to keep our employees safe during COVID-19, you are likely to experience longer than usual review times… https://t.co/Oo1t07UH4Z", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, "source": 'Twitter Web App', - "text": "RT @TwitterDev: As we work to keep our employees safe during COVID-19, you are likely to experience longer than usual review times for deve…", "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, @@ -377,309 +472,67 @@ example_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Mon Mar 23 16:28:09 +0000 2020", + "created_at": "Wed Aug 05 19:31:23 +0000 2020", + "display_text_range": [0, 218], "entities": { "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ + "media": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "display_url": "pic.twitter.com/ESB5UBwhmO", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291094468473393152/photo/1", + "id": 1291090033080856576, + "id_str": "1291090033080856576", + "indices": [219, 242], + "media_url": "http://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", + "media_url_https": "https://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", + "sizes": { + "large": {"h": 635, "resize": "fit", "w": 1680}, + "medium": {"h": 454, "resize": "fit", "w": 1200}, + "small": {"h": 257, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/ESB5UBwhmO", } ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1242125997081673728, - "id_str": "1242125997081673728", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": True, - "lang": "en", - "place": None, - "quoted_status_id": 1204497856679202816, - "quoted_status_id_str": "1204497856679202816", - "retweet_count": 41, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Mon Mar 23 16:26:08 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1242125486844604425", - "indices": [117, 140], - "url": "https://t.co/ViHyvQ4Y8S", - } - ], - "user_mentions": [], - }, - "favorite_count": 120, - "favorited": False, - "geo": None, - "id": 1242125486844604425, - "id_str": "1242125486844604425", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": True, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "quoted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Tue Dec 10 20:27:22 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1204497856679202816", - "indices": [117, 140], - "url": "https://t.co/ZlJUjmHIBe", - } - ], - "user_mentions": [], - }, - "favorite_count": 255, - "favorited": False, - "geo": None, - "id": 1204497856679202816, - "id_str": "1204497856679202816", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 95, - "retweeted": False, - "source": 'Twitter Web App', - "text": "You may have seen the recent announcement about following Topics on Twitter. Today, we’re excited to provide API su… https://t.co/ZlJUjmHIBe", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "quoted_status_id": 1204497856679202816, - "quoted_status_id_str": "1204497856679202816", - "retweet_count": 41, - "retweeted": False, - "source": 'Twitter Web App', - "text": "A few months ago, we added Tweet annotations to the Labs’ streaming endpoints. These annotations help uncover detai… https://t.co/ViHyvQ4Y8S", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: A few months ago, we added Tweet annotations to the Labs’ streaming endpoints. These annotations help uncover details about…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Tue Mar 10 17:57:58 +0000 2020", - "entities": { - "hashtags": [], "symbols": [], - "urls": [], - "user_mentions": [ + "urls": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "display_url": "robertsspaceindustries.com/spectrum/commu\u2026", + "expanded_url": "https://robertsspaceindustries.com/spectrum/community/SC/forum/3/thread/where-in-the-verse-is-pico-screenshot-contest", + "indices": [195, 218], + "url": "https://t.co/jFU7YLniDR", } ], + "user_mentions": [], }, - "favorite_count": 0, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/ESB5UBwhmO", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291094468473393152/photo/1", + "id": 1291090033080856576, + "id_str": "1291090033080856576", + "indices": [219, 242], + "media_url": "http://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", + "media_url_https": "https://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", + "sizes": { + "large": {"h": 635, "resize": "fit", "w": 1680}, + "medium": {"h": 454, "resize": "fit", "w": 1200}, + "small": {"h": 257, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/ESB5UBwhmO", + } + ] + }, + "favorite_count": 306, "favorited": False, + "full_text": "Alpha 3.10 is live and our fuzzy friend Pico wants in on the action. \ud83d\udc27 We want to see what you and Pico are getting up to in the 'verse with our \"Where in the 'Verse is Pico?\" contest!\n\nDetails: https://t.co/jFU7YLniDR https://t.co/ESB5UBwhmO", "geo": None, - "id": 1237437557337513984, - "id_str": "1237437557337513984", + "id": 1291094468473393152, + "id_str": "1291094468473393152", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -688,184 +541,166 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 27, + "possibly_sensitive": False, + "retweet_count": 65, "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Tue Mar 10 17:47:53 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1237435017883762689", - "indices": [117, 140], - "url": "https://t.co/NpF4h9DaSq", - } - ], - "user_mentions": [], - }, - "favorite_count": 97, - "favorited": False, - "geo": None, - "id": 1237435017883762689, - "id_str": "1237435017883762689", - "in_reply_to_screen_name": "TwitterDev", - "in_reply_to_status_id": 1237435016134656006, - "in_reply_to_status_id_str": "1237435016134656006", - "in_reply_to_user_id": 2244994945, - "in_reply_to_user_id_str": "2244994945", - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 27, - "retweeted": False, - "source": 'Twitter Web App', - "text": "We ❤️ the incredible research people do using Twitter data to study topics like spam, abuse, and other areas relate… https://t.co/NpF4h9DaSq", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, "source": 'Twitter Web App', - "text": "RT @TwitterDev: We ❤️ the incredible research people do using Twitter data to study topics like spam, abuse, and other areas related to the…", "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, }, +] + +video_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Wed Feb 26 17:33:41 +0000 2020", + "created_at": "Wed Aug 05 18:36:00 +0000 2020", + "display_text_range": [0, 196], "entities": { "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ + "media": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/mZ8CAuq3SH", } ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/greycatroc", + "expanded_url": "http://robertsspaceindustries.com/greycatroc", + "indices": [173, 196], + "url": "https://t.co/2aH7qdOfSk", + } + ], + "user_mentions": [], }, - "favorite_count": 0, + "extended_entities": { + "media": [ + { + "additional_media_info": { + "description": "", + "embeddable": True, + "monetizable": False, + "title": "", + }, + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/mZ8CAuq3SH", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 82967, + "variants": [ + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/pl/kMYgFEoRyoW99o-i.m3u8?tag=13", + }, + { + "bitrate": 2176000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/vid/1280x720/J05_p6q74ZUN4csg.mp4?tag=13", + }, + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/vid/640x360/ya3fVKeRdBs3cOoF.mp4?tag=13", + }, + { + "bitrate": 288000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/vid/480x270/WQkAozOts-hRoU1I.mp4?tag=13", + }, + ], + }, + } + ] + }, + "favorite_count": 289, "favorited": False, + "full_text": "Small enough to access hard-to-reach ore deposits, but with enough power to get through the tough jobs, Greycat\u2019s ROC perfectly complements any mining operation. \n\nDetails: https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH", "geo": None, - "id": 1232720402700521474, - "id_str": "1232720402700521474", + "id": 1291080532361527296, + "id_str": "1291080532361527296", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -874,157 +709,64 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 148, + "possibly_sensitive": False, + "retweet_count": 64, "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Wed Feb 26 17:32:51 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1232720193182412800", - "indices": [117, 140], - "url": "https://t.co/aN8kan0Lsw", - } - ], - "user_mentions": [], - }, - "favorite_count": 386, - "favorited": False, - "geo": None, - "id": 1232720193182412800, - "id_str": "1232720193182412800", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 148, - "retweeted": False, - "source": 'Twitter Web App', - "text": "In November, we gave people the ability to hide replies to their Tweets. Starting today, we’re opening this feature… https://t.co/aN8kan0Lsw", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: In November, we gave people the ability to hide replies to their Tweets. Starting today, we’re opening this feature up to d…", + "source": 'Twitter Media Studio', "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, @@ -1032,26 +774,98 @@ example_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Mon Jan 06 20:23:49 +0000 2020", + "created_at": "Wed Aug 05 18:31:27 +0000 2020", + "display_text_range": [0, 213], "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ + "hashtags": [{"indices": [157, 169], "text": "StarCitizen"}], + "media": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "display_url": "pic.twitter.com/lri5QijMoA", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291079386821582849/video/1", + "id": 1291070740347813889, + "id_str": "1291070740347813889", + "indices": [214, 237], + "media_url": "http://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/lri5QijMoA", } ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/comm-link/tran\u2026", + "expanded_url": "https://robertsspaceindustries.com/comm-link/transmission/17648-Alpha-310-Flight-Fight", + "indices": [190, 213], + "url": "https://t.co/6jT1yuZMiR", + } + ], + "user_mentions": [], }, - "favorite_count": 0, + "extended_entities": { + "media": [ + { + "additional_media_info": { + "description": "", + "embeddable": True, + "monetizable": False, + "title": "", + }, + "display_url": "pic.twitter.com/lri5QijMoA", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291079386821582849/video/1", + "id": 1291070740347813889, + "id_str": "1291070740347813889", + "indices": [214, 237], + "media_url": "http://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerUMgyUwAAgj9w.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/lri5QijMoA", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 83633, + "variants": [ + { + "bitrate": 288000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/vid/480x270/oGdSeLr5QQ-XcTns.mp4?tag=13", + }, + { + "bitrate": 2176000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/vid/1280x720/bql0evKsgYZhGPNP.mp4?tag=13", + }, + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/vid/640x360/lSL6mqB53HnwrUo4.mp4?tag=13", + }, + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/amplify_video/1291070740347813889/pl/_jJ-AYWSMr8ZS1WP.m3u8?tag=13", + }, + ], + }, + } + ] + }, + "favorite_count": 429, "favorited": False, + "full_text": "Harness the power of improved high-speed dynamic combat. Feel the thrill of atmospheric flight like never before. Alpha 3.10 will change the way you play. \ud83d\ude80 #StarCitizen\n\nGet in the 'verse: https://t.co/6jT1yuZMiR https://t.co/lri5QijMoA", "geo": None, - "id": 1214281438092238855, - "id_str": "1214281438092238855", + "id": 1291079386821582849, + "id_str": "1291079386821582849", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -1060,1300 +874,96 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 59, + "possibly_sensitive": False, + "retweet_count": 117, "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Mon Jan 06 20:22:05 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1214281000932593667", - "indices": [117, 140], - "url": "https://t.co/XaqD1JJ5kF", - } - ], - "user_mentions": [], - }, - "favorite_count": 166, - "favorited": False, - "geo": None, - "id": 1214281000932593667, - "id_str": "1214281000932593667", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 59, - "retweeted": False, - "source": 'Twitter Web App', - "text": "Hello… is it me you’re searching for? 🔎\n\nSearch the conversation as it unfolds with this new addition to Labs. We'r… https://t.co/XaqD1JJ5kF", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: Hello… is it me you’re searching for? 🔎\n\nSearch the conversation as it unfolds with this new addition to Labs. We're making…", + "source": 'Twitter Media Studio', "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, }, +] + +retweet_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Fri Jan 03 17:21:34 +0000 2020", + "created_at": "Wed Aug 05 21:01:02 +0000 2020", + "display_text_range": [0, 140], "entities": { - "hashtags": [], + "hashtags": [{"indices": [27, 39], "text": "StarCitizen"}], "symbols": [], "urls": [], "user_mentions": [ { - "id": 2244994945, - "id_str": "2244994945", + "id": 859293278100914176, + "id_str": "859293278100914176", "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", + "name": "Aleksandr Belov", + "screen_name": "Narayan_N7", } ], }, "favorite_count": 0, "favorited": False, + "full_text": "RT @Narayan_N7: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPle\u2026", "geo": None, - "id": 1213148410145992704, - "id_str": "1213148410145992704", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 218, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Fri Jan 03 17:17:23 +0000 2020", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1213147357551816704", - "indices": [117, 140], - "url": "https://t.co/WhV7rP54GM", - } - ], - "user_mentions": [], - }, - "favorite_count": 450, - "favorited": False, - "geo": None, - "id": 1213147357551816704, - "id_str": "1213147357551816704", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 218, - "retweeted": False, - "source": 'Twitter Web App', - "text": "Today, we’re sharing a few small improvements to make it easier for academic researchers to get started with the Tw… https://t.co/WhV7rP54GM", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: Today, we’re sharing a few small improvements to make it easier for academic researchers to get started with the Twitter AP…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Tue Dec 10 20:34:57 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1204499768459661312, - "id_str": "1204499768459661312", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 95, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Tue Dec 10 20:27:22 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1204497856679202816", - "indices": [117, 140], - "url": "https://t.co/ZlJUjmHIBe", - } - ], - "user_mentions": [], - }, - "favorite_count": 255, - "favorited": False, - "geo": None, - "id": 1204497856679202816, - "id_str": "1204497856679202816", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 95, - "retweeted": False, - "source": 'Twitter Web App', - "text": "You may have seen the recent announcement about following Topics on Twitter. Today, we’re excited to provide API su… https://t.co/ZlJUjmHIBe", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: You may have seen the recent announcement about following Topics on Twitter. Today, we’re excited to provide API support fo…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Tue Oct 29 19:39:40 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1189265562821640193, - "id_str": "1189265562821640193", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 46, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Tue Oct 29 19:37:15 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1189264953368338432", - "indices": [110, 133], - "url": "https://t.co/Y5QMh8rNoh", - } - ], - "user_mentions": [], - }, - "favorite_count": 145, - "favorited": False, - "geo": None, - "id": 1189264953368338432, - "id_str": "1189264953368338432", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 46, - "retweeted": False, - "source": 'Twitter Web App', - "text": "Study a sample of timely, relevant Tweets as they happen, with the newest release in Twitter Developer Labs.… https://t.co/Y5QMh8rNoh", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: Study a sample of timely, relevant Tweets as they happen, with the newest release in Twitter Developer Labs. https://t.co/m…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Wed Sep 18 16:41:39 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1174362863118372867, - "id_str": "1174362863118372867", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 106, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Wed Sep 18 16:36:10 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1174361480734466048", - "indices": [117, 140], - "url": "https://t.co/RElmBW5XxM", - } - ], - "user_mentions": [], - }, - "favorite_count": 234, - "favorited": False, - "geo": None, - "id": 1174361480734466048, - "id_str": "1174361480734466048", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 106, - "retweeted": False, - "source": 'Twitter Web App', - "text": "Today in Twitter Developer Labs we’re releasing a new way to filter Tweets in real-time. This is one of the most po… https://t.co/RElmBW5XxM", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: Today in Twitter Developer Labs we’re releasing a new way to filter Tweets in real-time. This is one of the most popular fe…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Tue Aug 27 17:30:39 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1166402661282746368, - "id_str": "1166402661282746368", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 59, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Tue Aug 27 17:25:06 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1166401263170281472", - "indices": [117, 140], - "url": "https://t.co/a8PaA1wg5A", - } - ], - "user_mentions": [], - }, - "favorite_count": 174, - "favorited": False, - "geo": None, - "id": 1166401263170281472, - "id_str": "1166401263170281472", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 59, - "retweeted": False, - "source": 'Twitter Web App', - "text": "Our latest Twitter Developer Labs release helps you quickly assess the impact of your Tweets. Today, we’re releasin… https://t.co/a8PaA1wg5A", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: Our latest Twitter Developer Labs release helps you quickly assess the impact of your Tweets. Today, we’re releasing \n\n✨a n…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Fri Aug 16 17:25:53 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1162415194749882368, - "id_str": "1162415194749882368", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 31, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Fri Aug 16 17:07:22 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1162410535121387525", - "indices": [117, 140], - "url": "https://t.co/VemT1Licd9", - } - ], - "user_mentions": [], - }, - "favorite_count": 102, - "favorited": False, - "geo": None, - "id": 1162410535121387525, - "id_str": "1162410535121387525", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 31, - "retweeted": False, - "source": 'Twitter Web App', - "text": "As of today, we’re simplifying permissions for third-party apps. Most developers won't be impacted, but if your app… https://t.co/VemT1Licd9", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] - }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter for iPhone', - "text": "RT @TwitterDev: As of today, we’re simplifying permissions for third-party apps. Most developers won't be impacted, but if your app uses th…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Thu Aug 15 16:24:23 +0000 2019", - "entities": { - "hashtags": [{"indices": [16, 26], "text": "iterating"}], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1162037328442875904, - "id_str": "1162037328442875904", + "id": 1291117030486106112, + "id_str": "1291117030486106112", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -2367,393 +977,42 @@ example_mock = [ "retweeted_status": { "contributors": None, "coordinates": None, - "created_at": "Thu Aug 15 16:20:05 +0000 2019", + "created_at": "Wed Aug 05 18:15:34 +0000 2020", + "display_text_range": [0, 250], "entities": { - "hashtags": [{"indices": [0, 10], "text": "iterating"}], + "hashtags": [{"indices": [11, 23], "text": "StarCitizen"}], "symbols": [], "urls": [ { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1162036247314833408", - "indices": [117, 140], - "url": "https://t.co/rRlVHaBTs2", + "display_url": "youtu.be/aXXGnCbEas0", + "expanded_url": "https://youtu.be/aXXGnCbEas0", + "indices": [227, 250], + "url": "https://t.co/j4QahHzbw4", } ], - "user_mentions": [], - }, - "favorite_count": 102, - "favorited": False, - "geo": None, - "id": 1162036247314833408, - "id_str": "1162036247314833408", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 26, - "retweeted": False, - "source": 'Twitter Web App', - "text": "#iterating: We recently released a long-requested feature in Twitter Developer Labs: \n\n📌Developer’s can now request… https://t.co/rRlVHaBTs2", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] + "user_mentions": [ + { + "id": 803542770, + "id_str": "803542770", + "indices": [193, 209], + "name": "Star Citizen", + "screen_name": "RobertsSpaceInd", }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: #iterating: We recently released a long-requested feature in Twitter Developer Labs: \n\n📌Developer’s can now request a user’…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Wed Jul 31 19:23:15 +0000 2019", - "entities": { - "hashtags": [{"indices": [16, 26], "text": "iterating"}], - "symbols": [], - "urls": [], - "user_mentions": [ - { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } - ], - }, - "favorite_count": 0, - "favorited": False, - "geo": None, - "id": 1156646525121916928, - "id_str": "1156646525121916928", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 41, - "retweeted": False, - "retweeted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Wed Jul 31 19:20:25 +0000 2019", - "entities": { - "hashtags": [{"indices": [0, 10], "text": "iterating"}], - "symbols": [], - "urls": [ { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1156645810156650496", - "indices": [117, 140], - "url": "https://t.co/VVp7rv6FIM", - } - ], - "user_mentions": [], - }, - "favorite_count": 111, - "favorited": False, - "geo": None, - "id": 1156645810156650496, - "id_str": "1156645810156650496", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "retweet_count": 41, - "retweeted": False, - "source": 'Twitter Web App', - "text": "#iterating: Today we are releasing an update to Twitter Developer Labs, with a few new features we think you’ll fin… https://t.co/VVp7rv6FIM", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", - "default_profile": False, - "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", - "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", - } - ] + "id": 803697073, + "id_str": "803697073", + "indices": [211, 225], + "name": "Cloud Imperium Games", + "screen_name": "CloudImperium", }, - }, - "favourites_count": 2165, - "follow_request_sent": None, - "followers_count": 509354, - "following": None, - "friends_count": 2000, - "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", - "notifications": None, - "profile_background_color": "FFFFFF", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, - "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", - "utc_offset": None, - "verified": True, - }, - }, - "source": 'Twitter Web App', - "text": "RT @TwitterDev: #iterating: Today we are releasing an update to Twitter Developer Labs, with a few new features we think you’ll find useful…", - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Wed Jul 24 15:56:09 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1154057692723519494", - "indices": [117, 140], - "url": "https://t.co/8YgCwYoE3q", - } - ], - "user_mentions": [], - }, - "favorite_count": 128, - "favorited": False, - "geo": None, - "id": 1154057692723519494, - "id_str": "1154057692723519494", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": True, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "quoted_status": { - "contributors": None, - "coordinates": None, - "created_at": "Tue Jun 11 22:13:27 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1138569964032385025", - "indices": [117, 140], - "url": "https://t.co/qMtoumuG1e", - } ], - "user_mentions": [], }, - "favorite_count": 121, + "favorite_count": 97, "favorited": False, + "full_text": "New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease, share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe! \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\nhttps://t.co/j4QahHzbw4", "geo": None, - "id": 1138569964032385025, - "id_str": "1138569964032385025", + "id": 1291075388798533633, + "id_str": "1291075388798533633", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -2763,128 +1022,131 @@ example_mock = [ "lang": "en", "place": None, "possibly_sensitive": False, - "retweet_count": 91, + "retweet_count": 26, "retweeted": False, - "source": 'Twitter Web Client', - "text": "Starting July 15, 2019, all connections to the Twitter API (and all other Twitter domains) will require TLS 1.2. Re… https://t.co/qMtoumuG1e", - "truncated": True, + "source": 'Twitter Web App', + "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, + "created_at": "Tue May 02 06:27:37 +0000 2017", + "default_profile": True, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "Enlist to Star Citizen: https://t.co/JOei50wjGK Content creator. #IWantToWorkAtCIG \n#StarCitizen #video #youtube #flickr #4K #panorama", "entities": { - "description": {"urls": []}, + "description": { + "urls": [ + { + "display_url": "goo.gl/8CbEZm", + "expanded_url": "http://goo.gl/8CbEZm", + "indices": [24, 47], + "url": "https://t.co/JOei50wjGK", + } + ] + }, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "youtube.com/user/sashaMOHC\u2026", + "expanded_url": "https://www.youtube.com/user/sashaMOHCTPwhite", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/ise14uN9Ja", } ] }, }, - "favourites_count": 3, + "favourites_count": 1882, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 489, "following": None, - "friends_count": 32, - "geo_enabled": False, + "friends_count": 80, + "geo_enabled": True, "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "id": 859293278100914176, + "id_str": "859293278100914176", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 16, + "location": "\u0421\u0430\u043d\u043a\u0442-\u041f\u0435\u0442\u0435\u0440\u0431\u0443\u0440\u0433, \u0420\u043e\u0441\u0441\u0438\u044f", + "name": "Aleksandr Belov", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", + "profile_background_color": "F5F8FA", + "profile_background_image_url": None, + "profile_background_image_url_https": None, + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/859293278100914176/1576841460", + "profile_image_url": "http://pbs.twimg.com/profile_images/1203066581573607425/5TEkxVJ3_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1203066581573607425/5TEkxVJ3_normal.jpg", + "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "Narayan_N7", + "statuses_count": 1283, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/ise14uN9Ja", "utc_offset": None, - "verified": True, + "verified": False, }, }, - "quoted_status_id": 1138569964032385025, - "quoted_status_id_str": "1138569964032385025", - "retweet_count": 108, - "retweeted": False, "source": 'Twitter Web App', - "text": "TLS 1.2 reminder: this change will be enacted as of tomorrow, July 25, 2019. Please reference our developer forum p… https://t.co/8YgCwYoE3q", - "truncated": True, + "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, @@ -2892,26 +1154,35 @@ example_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Wed Jul 17 15:54:45 +0000 2019", + "created_at": "Thu Jul 30 13:15:25 +0000 2020", + "display_text_range": [0, 140], "entities": { - "hashtags": [], + "hashtags": [{"indices": [24, 40], "text": "CountdownToMars"}], "symbols": [], "urls": [], "user_mentions": [ { - "id": 2244994945, - "id_str": "2244994945", - "indices": [3, 14], - "name": "Twitter Dev", - "screen_name": "TwitterDev", - } + "id": 11348282, + "id_str": "11348282", + "indices": [3, 8], + "name": "NASA", + "screen_name": "NASA", + }, + { + "id": 1232783237623119872, + "id_str": "1232783237623119872", + "indices": [123, 137], + "name": "NASA's Perseverance Mars Rover", + "screen_name": "NASAPersevere", + }, ], }, "favorite_count": 0, "favorited": False, + "full_text": "RT @NASA: LIVE NOW: The #CountdownToMars begins. \n\nWe are launching a historic mission to the Red Planet. Tune in to watch @NASAPersevere l\u2026", "geo": None, - "id": 1151520624315174912, - "id_str": "1151520624315174912", + "id": 1288825524878336000, + "id_str": "1288825524878336000", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -2920,30 +1191,40 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 421, + "retweet_count": 8867, "retweeted": False, "retweeted_status": { "contributors": None, "coordinates": None, - "created_at": "Wed Jul 17 15:53:43 +0000 2019", + "created_at": "Thu Jul 30 11:01:06 +0000 2020", + "display_text_range": [0, 236], "entities": { - "hashtags": [], + "hashtags": [{"indices": [14, 30], "text": "CountdownToMars"}], "symbols": [], "urls": [ { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1151520361529430016", - "indices": [117, 140], - "url": "https://t.co/dG3PmGWAJ4", + "display_url": "twitter.com/i/broadcasts/1\u2026", + "expanded_url": "https://twitter.com/i/broadcasts/1RDGlrkoEzNxL", + "indices": [213, 236], + "url": "https://t.co/JxyRCol01i", + } + ], + "user_mentions": [ + { + "id": 1232783237623119872, + "id_str": "1232783237623119872", + "indices": [113, 127], + "name": "NASA's Perseverance Mars Rover", + "screen_name": "NASAPersevere", } ], - "user_mentions": [], }, - "favorite_count": 675, + "favorite_count": 18327, "favorited": False, + "full_text": "LIVE NOW: The #CountdownToMars begins. \n\nWe are launching a historic mission to the Red Planet. Tune in to watch @NASAPersevere liftoff and begin her mission to search for signs of ancient life on another world: https://t.co/JxyRCol01i", "geo": None, - "id": 1151520361529430016, - "id_str": "1151520361529430016", + "id": 1288791726165983233, + "id_str": "1288791726165983233", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -2952,217 +1233,388 @@ example_mock = [ "is_quote_status": False, "lang": "en", "place": None, - "retweet_count": 421, + "possibly_sensitive": False, + "retweet_count": 8867, "retweeted": False, - "source": 'Twitter Web App', - "text": "Academic research is some of the most impactful work that happens with the Twitter API. As we plan for the future o… https://t.co/dG3PmGWAJ4", - "truncated": True, + "source": 'Twitter Media Studio', + "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "created_at": "Wed Dec 19 20:20:32 +0000 2007", "default_profile": False, "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "description": "Explore the universe and our home planet with NASA \ud83c\udf0e We usually post in EDT.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", + "display_url": "nasa.gov", + "expanded_url": "http://www.nasa.gov/", "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", + "url": "https://t.co/HMJJbimQpV", } ] }, }, - "favourites_count": 2165, + "favourites_count": 11658, "follow_request_sent": None, - "followers_count": 509354, + "followers_count": 39440029, "following": None, - "friends_count": 2000, - "geo_enabled": True, + "friends_count": 222, + "geo_enabled": False, "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", + "id": 11348282, + "id_str": "11348282", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", + "listed_count": 92535, + "location": "", + "name": "NASA", "notifications": None, - "profile_background_color": "FFFFFF", + "profile_background_color": "000000", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/11348282/1596217000", + "profile_image_url": "http://pbs.twimg.com/profile_images/1091070803184177153/TI2qItoi_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1091070803184177153/TI2qItoi_normal.jpg", + "profile_link_color": "205BA7", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "F3F2F2", + "profile_text_color": "000000", + "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, + "screen_name": "NASA", + "statuses_count": 61920, "time_zone": None, "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", + "url": "https://t.co/HMJJbimQpV", "utc_offset": None, "verified": True, }, }, - "source": 'TweetDeck', - "text": "RT @TwitterDev: Academic research is some of the most impactful work that happens with the Twitter API. As we plan for the future of our de…", + "source": 'Twitter for iPhone', "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, }, +] + +quoted_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Mon Jun 24 17:50:46 +0000 2019", + "created_at": "Wed Aug 05 00:05:24 +0000 2020", + "display_text_range": [0, 13], "entities": { "hashtags": [], "symbols": [], "urls": [ { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1143214899109277697", - "indices": [117, 140], - "url": "https://t.co/w8WoepBjeU", + "display_url": "twitter.com/hugolisoir/sta\u2026", + "expanded_url": "https://twitter.com/hugolisoir/status/1290778178793897992", + "indices": [14, 37], + "url": "https://t.co/WyznJwCJLp", } ], "user_mentions": [], }, - "favorite_count": 85, + "favorite_count": 576, "favorited": False, + "full_text": "Bonne nuit \ud83c\udf3a\ud83d\udeeb https://t.co/WyznJwCJLp", "geo": None, - "id": 1143214899109277697, - "id_str": "1143214899109277697", - "in_reply_to_screen_name": "TwitterAPI", - "in_reply_to_status_id": 1141392777600806912, - "in_reply_to_status_id_str": "1141392777600806912", - "in_reply_to_user_id": 6253282, - "in_reply_to_user_id_str": "6253282", - "is_quote_status": False, - "lang": "en", + "id": 1290801039075979264, + "id_str": "1290801039075979264", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": True, + "lang": "fr", "place": None, "possibly_sensitive": False, - "retweet_count": 44, + "quoted_status": { + "contributors": None, + "coordinates": None, + "created_at": "Tue Aug 04 22:34:33 +0000 2020", + "display_text_range": [0, 57], + "entities": { + "hashtags": [{"indices": [0, 12], "text": "Starcitizen"}], + "media": [ + { + "display_url": "pic.twitter.com/xCXun68V3r", + "expanded_url": "https://twitter.com/hugolisoir/status/1290778178793897992/video/1", + "id": 1290778053623382017, + "id_str": "1290778053623382017", + "indices": [58, 81], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/xCXun68V3r", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 803542770, + "id_str": "803542770", + "indices": [41, 57], + "name": "Star Citizen", + "screen_name": "RobertsSpaceInd", + } + ], + }, + "extended_entities": { + "media": [ + { + "additional_media_info": {"monetizable": False}, + "display_url": "pic.twitter.com/xCXun68V3r", + "expanded_url": "https://twitter.com/hugolisoir/status/1290778178793897992/video/1", + "id": 1290778053623382017, + "id_str": "1290778053623382017", + "indices": [58, 81], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1290778053623382017/pu/img/FFHKsCa_gYLNrriu.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/xCXun68V3r", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 39901, + "variants": [ + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/vid/640x360/jYjO0H2SYSycTi-e.mp4?tag=10", + }, + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/pl/wFnVMLjVWi7OKy2o.m3u8?tag=10", + }, + { + "bitrate": 2176000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/vid/1280x720/H-BXvYdM0AcSKXpk.mp4?tag=10", + }, + { + "bitrate": 256000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1290778053623382017/pu/vid/480x270/aWhSjP1gK7djKZUK.mp4?tag=10", + }, + ], + }, + } + ] + }, + "favorite_count": 400, + "favorited": False, + "full_text": "#Starcitizen Le jeu est beau. Bonne nuit @RobertsSpaceInd https://t.co/xCXun68V3r", + "geo": None, + "id": 1290778178793897992, + "id_str": "1290778178793897992", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "fr", + "place": None, + "possibly_sensitive": False, + "retweet_count": 76, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Tue Mar 22 12:00:36 +0000 2011", + "default_profile": False, + "default_profile_image": False, + "description": "Youtuber Partner / Twitch Partner / Membre du @CurryClub_CC\nInsta - hugolisoir\nParrain de @AbyssalProject", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "youtube.com/channel/UCDC6D\u2026", + "expanded_url": "https://www.youtube.com/channel/UCDC6DBi0kRp6Jk21xqfvFLA", + "indices": [0, 23], + "url": "https://t.co/p3CVR2I068", + } + ] + }, + }, + "favourites_count": 20935, + "follow_request_sent": None, + "followers_count": 23269, + "following": None, + "friends_count": 703, + "geo_enabled": True, + "has_extended_profile": False, + "id": 270320632, + "id_str": "270320632", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 116, + "location": "Nantes, France", + "name": "Hugo Lisoir #ZLAN2020", + "notifications": None, + "profile_background_color": "000000", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme15/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme15/bg.png", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/270320632/1499086260", + "profile_image_url": "http://pbs.twimg.com/profile_images/1264841251305730048/vyUJVCvW_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1264841251305730048/vyUJVCvW_normal.jpg", + "profile_link_color": "ABB8C2", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "000000", + "profile_text_color": "000000", + "profile_use_background_image": False, + "protected": False, + "screen_name": "hugolisoir", + "statuses_count": 7507, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/p3CVR2I068", + "utc_offset": None, + "verified": False, + }, + }, + "quoted_status_id": 1290778178793897992, + "quoted_status_id_str": "1290778178793897992", + "quoted_status_permalink": { + "display": "twitter.com/hugolisoir/sta\u2026", + "expanded": "https://twitter.com/hugolisoir/status/1290778178793897992", + "url": "https://t.co/WyznJwCJLp", + }, + "retweet_count": 60, "retweeted": False, - "source": 'Twitter Web Client', - "text": "We’ve spoken with all developers who’ve contacted us to discuss these new rate limits and elevations, and as of tod… https://t.co/w8WoepBjeU", - "truncated": True, + "source": 'Twitter Web App', + "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, @@ -3170,25 +1622,27 @@ example_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Wed Jun 19 17:10:18 +0000 2019", + "created_at": "Fri Jul 31 22:00:55 +0000 2020", + "display_text_range": [0, 32], "entities": { "hashtags": [], "symbols": [], "urls": [ { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1141392777600806912", - "indices": [117, 140], - "url": "https://t.co/ymDvv7r8lB", + "display_url": "twitter.com/UberFacts/stat\u2026", + "expanded_url": "https://twitter.com/UberFacts/status/1289273883493675009", + "indices": [33, 56], + "url": "https://t.co/LLPVr8oU7F", } ], "user_mentions": [], }, - "favorite_count": 90, + "favorite_count": 263, "favorited": False, + "full_text": "Here's to our lovely Avocados! \ud83d\udd79 https://t.co/LLPVr8oU7F", "geo": None, - "id": 1141392777600806912, - "id_str": "1141392777600806912", + "id": 1289320160021495809, + "id_str": "1289320160021495809", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, @@ -3201,344 +1655,497 @@ example_mock = [ "quoted_status": { "contributors": None, "coordinates": None, - "created_at": "Wed Jun 19 17:02:25 +0000 2019", + "created_at": "Fri Jul 31 18:57:02 +0000 2020", + "display_text_range": [0, 34], "entities": { "hashtags": [], - "symbols": [], - "urls": [ + "media": [ { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1141390793657266176", - "indices": [117, 140], - "url": "https://t.co/kFzCKybdkD", + "display_url": "pic.twitter.com/8QRycx9QB2", + "expanded_url": "https://twitter.com/UberFacts/status/1289273883493675009/photo/1", + "id": 1289273880570363907, + "id_str": "1289273880570363907", + "indices": [35, 58], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "sizes": { + "large": {"h": 500, "resize": "fit", "w": 500}, + "medium": {"h": 500, "resize": "fit", "w": 500}, + "small": {"h": 500, "resize": "fit", "w": 500}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/8QRycx9QB2", } ], + "symbols": [], + "urls": [], "user_mentions": [], }, - "favorite_count": 46, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/8QRycx9QB2", + "expanded_url": "https://twitter.com/UberFacts/status/1289273883493675009/photo/1", + "id": 1289273880570363907, + "id_str": "1289273880570363907", + "indices": [35, 58], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeRrw3WWAAMKVF0.jpg", + "sizes": { + "large": {"h": 500, "resize": "fit", "w": 500}, + "medium": {"h": 500, "resize": "fit", "w": 500}, + "small": {"h": 500, "resize": "fit", "w": 500}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "animated_gif", + "url": "https://t.co/8QRycx9QB2", + "video_info": { + "aspect_ratio": [1, 1], + "variants": [ + { + "bitrate": 0, + "content_type": "video/mp4", + "url": "https://video.twimg.com/tweet_video/EeRrw3WWAAMKVF0.mp4", + } + ], + }, + } + ] + }, + "favorite_count": 1550, "favorited": False, + "full_text": "July 31st is National Avocado Day! https://t.co/8QRycx9QB2", "geo": None, - "id": 1141390793657266176, - "id_str": "1141390793657266176", + "id": 1289273883493675009, + "id_str": "1289273883493675009", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, "in_reply_to_user_id": None, "in_reply_to_user_id_str": None, - "is_quote_status": True, + "is_quote_status": False, "lang": "en", "place": None, "possibly_sensitive": False, - "quoted_status_id": 1108050885639168000, - "quoted_status_id_str": "1108050885639168000", - "retweet_count": 19, + "retweet_count": 380, "retweeted": False, - "source": 'Twitter Web Client', - "text": "❗️Today, user and mentions timeline request limits go into effect. If you want to learn more about this change, tak… https://t.co/kFzCKybdkD", - "truncated": True, + "source": 'Buffer', + "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Sat Dec 14 04:35:55 +0000 2013", + "created_at": "Sun Dec 06 16:07:01 +0000 2009", "default_profile": False, "default_profile_image": False, - "description": "The voice of the #TwitterDev team and your official source for updates, news, and events, related to the #TwitterAPI.", + "description": "The most unimportant things you'll never need to know.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com/en/community", - "expanded_url": "https://developer.twitter.com/en/community", + "display_url": "uber-facts.com", + "expanded_url": "http://uber-facts.com/", "indices": [0, 23], - "url": "https://t.co/3ZX3TNiZCY", + "url": "https://t.co/3ycpGqEL9n", } ] }, }, - "favourites_count": 2165, + "favourites_count": 1297, "follow_request_sent": None, - "followers_count": 509354, + "followers_count": 13810392, "following": None, - "friends_count": 2000, + "friends_count": 1, "geo_enabled": True, - "has_extended_profile": True, - "id": 2244994945, - "id_str": "2244994945", - "is_translation_enabled": False, + "has_extended_profile": False, + "id": 95023423, + "id_str": "95023423", + "is_translation_enabled": True, "is_translator": False, "lang": None, - "listed_count": 1589, - "location": "127.0.0.1", - "name": "Twitter Dev", + "listed_count": 15141, + "location": "Worldwide!", + "name": "UberFacts", "notifications": None, - "profile_background_color": "FFFFFF", + "profile_background_color": "C0DEED", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1594913664", - "profile_image_url": "http://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1283786620521652229/lEODkLTh_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/95023423/1587338728", + "profile_image_url": "http://pbs.twimg.com/profile_images/615696617165885440/JDbUuo9H_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/615696617165885440/JDbUuo9H_normal.jpg", + "profile_link_color": "0D9BA8", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "FFFFFF", + "profile_text_color": "000000", + "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterDev", - "statuses_count": 3576, + "screen_name": "UberFacts", + "statuses_count": 202253, "time_zone": None, "translator_type": "regular", - "url": "https://t.co/3ZX3TNiZCY", + "url": "https://t.co/3ycpGqEL9n", "utc_offset": None, "verified": True, }, }, - "quoted_status_id": 1141390793657266176, - "quoted_status_id_str": "1141390793657266176", - "retweet_count": 70, + "quoted_status_id": 1289273883493675009, + "quoted_status_id_str": "1289273883493675009", + "quoted_status_permalink": { + "display": "twitter.com/UberFacts/stat\u2026", + "expanded": "https://twitter.com/UberFacts/status/1289273883493675009", + "url": "https://t.co/LLPVr8oU7F", + }, + "retweet_count": 24, "retweeted": False, - "source": 'Twitter Web Client', - "text": "Request limit change: today, we're implementing a change to two commonly used Twitter standard API endpoints - user… https://t.co/ymDvv7r8lB", - "truncated": True, + "source": 'Twitter Web App', + "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", + "created_at": "Wed Sep 05 00:58:11 +0000 2012", "default_profile": False, "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", "entities": { "description": {"urls": []}, "url": { "urls": [ { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", + "url": "https://t.co/iqO6apof3y", } ] }, }, - "favourites_count": 3, + "favourites_count": 4588, "follow_request_sent": None, - "followers_count": 6052538, + "followers_count": 106169, "following": None, - "friends_count": 32, + "friends_count": 201, "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "profile_use_background_image": True, "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Wed Jun 12 17:36:26 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1138862637394137093", - "indices": [116, 139], - "url": "https://t.co/JAUtpAZotb", - } - ], - "user_mentions": [], - }, - "favorite_count": 62, - "favorited": False, - "geo": None, - "id": 1138862637394137093, - "id_str": "1138862637394137093", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 40, - "retweeted": False, - "source": 'Twitter Web Client', - "text": "Reminder: only 1⃣ week until the rate limit change to user and mentions timeline endpoints will go into effect. If… https://t.co/JAUtpAZotb", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Tue Jun 11 22:13:27 +0000 2019", - "entities": { - "hashtags": [], - "symbols": [], - "urls": [ - { - "display_url": "twitter.com/i/web/status/1…", - "expanded_url": "https://twitter.com/i/web/status/1138569964032385025", - "indices": [117, 140], - "url": "https://t.co/qMtoumuG1e", - } - ], - "user_mentions": [], - }, - "favorite_count": 121, - "favorited": False, - "geo": None, - "id": 1138569964032385025, - "id_str": "1138569964032385025", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 91, - "retweeted": False, - "source": 'Twitter Web Client', - "text": "Starting July 15, 2019, all connections to the Twitter API (and all other Twitter domains) will require TLS 1.2. Re… https://t.co/qMtoumuG1e", - "truncated": True, - "user": { - "contributors_enabled": False, - "created_at": "Wed May 23 06:01:13 +0000 2007", - "default_profile": False, - "default_profile_image": False, - "description": "Tweets about changes and service issues. Follow @TwitterDev for more.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "developer.twitter.com", - "expanded_url": "https://developer.twitter.com", - "indices": [0, 23], - "url": "https://t.co/8IkCzCDr19", - } - ] - }, - }, - "favourites_count": 3, - "follow_request_sent": None, - "followers_count": 6052538, - "following": None, - "friends_count": 32, - "geo_enabled": False, - "has_extended_profile": True, - "id": 6253282, - "id_str": "6253282", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 12307, - "location": "", - "name": "Twitter API", - "notifications": None, - "profile_background_color": "C0DEED", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": True, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/6253282/1497491515", - "profile_image_url": "http://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/942858479592554497/BbazLO9L_normal.jpg", - "profile_link_color": "0084B4", - "profile_sidebar_border_color": "C0DEED", - "profile_sidebar_fill_color": "DDEEF6", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "TwitterAPI", - "statuses_count": 3679, - "time_zone": None, - "translator_type": "regular", - "url": "https://t.co/8IkCzCDr19", + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +gif_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Jul 31 23:10:55 +0000 2020", + "display_text_range": [12, 12], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/wxvioLCJ6h", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1289337776140296193/photo/1", + "id": 1289337769521606656, + "id_str": "1289337769521606656", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "sizes": { + "large": {"h": 210, "resize": "fit", "w": 250}, + "medium": {"h": 210, "resize": "fit", "w": 250}, + "small": {"h": 210, "resize": "fit", "w": 250}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/wxvioLCJ6h", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 994361231057346561, + "id_str": "994361231057346561", + "indices": [0, 12], + "name": "Xenosystems", + "screen_name": "Xenosystems", + } + ], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/wxvioLCJ6h", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1289337776140296193/photo/1", + "id": 1289337769521606656, + "id_str": "1289337769521606656", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeSl3sPUcAAyE4J.jpg", + "sizes": { + "large": {"h": 210, "resize": "fit", "w": 250}, + "medium": {"h": 210, "resize": "fit", "w": 250}, + "small": {"h": 210, "resize": "fit", "w": 250}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "animated_gif", + "url": "https://t.co/wxvioLCJ6h", + "video_info": { + "aspect_ratio": [25, 21], + "variants": [ + { + "bitrate": 0, + "content_type": "video/mp4", + "url": "https://video.twimg.com/tweet_video/EeSl3sPUcAAyE4J.mp4", + } + ], + }, + } + ] + }, + "favorite_count": 13, + "favorited": False, + "full_text": "@Xenosystems https://t.co/wxvioLCJ6h", + "geo": None, + "id": 1289337776140296193, + "id_str": "1289337776140296193", + "in_reply_to_screen_name": "Xenosystems", + "in_reply_to_status_id": 1289324787815178242, + "in_reply_to_status_id_str": "1289324787815178242", + "in_reply_to_user_id": 994361231057346561, + "in_reply_to_user_id_str": "994361231057346561", + "is_quote_status": False, + "lang": "und", + "place": None, + "possibly_sensitive": False, + "retweet_count": 1, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Thu Jul 30 22:30:29 +0000 2020", + "display_text_range": [12, 12], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/DTbhK1pTc4", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1288965215648849920/photo/1", + "id": 1288965209596420097, + "id_str": "1288965209596420097", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "sizes": { + "large": {"h": 278, "resize": "fit", "w": 498}, + "medium": {"h": 278, "resize": "fit", "w": 498}, + "small": {"h": 278, "resize": "fit", "w": 498}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/DTbhK1pTc4", + } + ], + "symbols": [], + "urls": [], + "user_mentions": [ + { + "id": 994361231057346561, + "id_str": "994361231057346561", + "indices": [0, 12], + "name": "Xenosystems", + "screen_name": "Xenosystems", + } + ], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/DTbhK1pTc4", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1288965215648849920/photo/1", + "id": 1288965209596420097, + "id_str": "1288965209596420097", + "indices": [13, 36], + "media_url": "http://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "media_url_https": "https://pbs.twimg.com/tweet_video_thumb/EeNTB2XU4AE-z5Y.jpg", + "sizes": { + "large": {"h": 278, "resize": "fit", "w": 498}, + "medium": {"h": 278, "resize": "fit", "w": 498}, + "small": {"h": 278, "resize": "fit", "w": 498}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "animated_gif", + "url": "https://t.co/DTbhK1pTc4", + "video_info": { + "aspect_ratio": [249, 139], + "variants": [ + { + "bitrate": 0, + "content_type": "video/mp4", + "url": "https://video.twimg.com/tweet_video/EeNTB2XU4AE-z5Y.mp4", + } + ], + }, + } + ] + }, + "favorite_count": 20, + "favorited": False, + "full_text": "@Xenosystems https://t.co/DTbhK1pTc4", + "geo": None, + "id": 1288965215648849920, + "id_str": "1288965215648849920", + "in_reply_to_screen_name": "Xenosystems", + "in_reply_to_status_id": 1288960722349719554, + "in_reply_to_status_id_str": "1288960722349719554", + "in_reply_to_user_id": 994361231057346561, + "in_reply_to_user_id_str": "994361231057346561", + "is_quote_status": False, + "lang": "und", + "place": None, + "possibly_sensitive": False, + "retweet_count": 0, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", "utc_offset": None, "verified": True, }, diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index e69de29..0488e95 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -0,0 +1,62 @@ +from unittest import skip + +from django.test import TestCase + + +class TwitterBuilderTestCase(TestCase): + def setUp(self): + self.maxDiff = None + + @skip("Not implemented") + def test_simple_post(self): + pass + + # Note that only one media type can be uploaded to an Tweet + # see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object + @skip("Not implemented") + def test_images_in_post(self): + pass + + @skip("Not implemented") + def test_videos_in_post(self): + pass + + @skip("Not implemented") + def test_GIFs_in_post(self): + pass + + @skip("Not implemented") + def test_retweet_post(self): + pass + + @skip("Not implemented") + def test_quoted_post(self): + pass + + @skip("Not implemented") + def test_images_in_quoted_post(self): + pass + + @skip("Not implemented") + def test_videos_in_quoted_post(self): + pass + + @skip("Not implemented") + def test_GIFs_in_quoted_post(self): + pass + + @skip("Not implemented") + def test_empty_data(self): + pass + + @skip("Not implemented") + def test_update_posts(self): + pass + + @skip("Not implemented") + def test_html_sanitizing(self): + pass + + @skip("Not implemented") + def test_duplicate_in_data(self): + pass From 90bbcb4d2750fa2dce569035d8526727cd91dd4f Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 7 Aug 2020 17:52:10 +0200 Subject: [PATCH 03/69] Add initial test --- .../news/collection/tests/factories.py | 4 ++ .../collection/tests/twitter/builder/tests.py | 41 ++++++++++++++++++- src/newsreader/news/collection/twitter.py | 3 ++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/tests/factories.py b/src/newsreader/news/collection/tests/factories.py index fdf786f..a84365d 100644 --- a/src/newsreader/news/collection/tests/factories.py +++ b/src/newsreader/news/collection/tests/factories.py @@ -28,3 +28,7 @@ class FeedFactory(CollectionRuleFactory): class SubredditFactory(CollectionRuleFactory): type = RuleTypeChoices.subreddit website_url = REDDIT_URL + + +class TwitterProfileFactory(CollectionRuleFactory): + type = RuleTypeChoices.twitter diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 0488e95..8e79913 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -1,6 +1,17 @@ +from datetime import datetime from unittest import skip +from unittest.mock import MagicMock +from django.template.defaultfilters import truncatechars from django.test import TestCase +from django.utils.html import format_html + +import pytz + +from newsreader.news.collection.tests.factories import TwitterProfileFactory +from newsreader.news.collection.tests.twitter.builder.mocks import simple_mock +from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder +from newsreader.news.core.models import Post class TwitterBuilderTestCase(TestCase): @@ -9,7 +20,35 @@ class TwitterBuilderTestCase(TestCase): @skip("Not implemented") def test_simple_post(self): - pass + builder = TwitterBuilder + + profile = TwitterProfileFactory() + mock_stream = MagicMock(rule=profile) + + with builder((simple_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291528756373286914", "1288550304095416320"), posts.keys() + ) + + post = posts["1291528756373286914"] + + full_text = "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, truncatechars(full_text, 20)) + self.assertEquals(post.body, format_html(full_text)) + + self.assertEquals(post.author, "Star Citizen") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291528756373286914" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 8, 7, 0, 17, 5)) + ) # Note that only one media type can be uploaded to an Tweet # see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 15a2ccd..5061e38 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -1,6 +1,9 @@ from newsreader.news.collection.base import Builder, Client, Collector, Stream +TWITTER_URL = "https://twitter.com" + + class TwitterScheduler: pass From f5d1e9ba5f46ace06b801c98a040099c6d106cef Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 7 Aug 2020 21:10:23 +0200 Subject: [PATCH 04/69] Add simple builder scenario --- src/newsreader/news/collection/base.py | 16 ++++++- src/newsreader/news/collection/feed.py | 19 +------- .../migrations/0009_auto_20200807_2030.py | 29 ++++++++++++ src/newsreader/news/collection/models.py | 4 ++ src/newsreader/news/collection/reddit.py | 17 +------ .../news/collection/tests/factories.py | 1 + .../collection/tests/twitter/builder/tests.py | 23 +++++++-- src/newsreader/news/collection/twitter.py | 47 +++++++++++++++---- 8 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index f980191..5de3454 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -2,6 +2,7 @@ from bs4 import BeautifulSoup from newsreader.news.collection.exceptions import StreamParseException from newsreader.news.collection.utils import fetch +from newsreader.news.core.models import Post class Stream: @@ -54,22 +55,33 @@ class Builder: instances = [] stream = None + rule_type = None def __init__(self, stream): self.stream = stream def __enter__(self): + _, stream = self.stream + + self.instances = [] + self.existing_posts = { + post.remote_identifier: post + for post in Post.objects.filter(rule=stream.rule, rule__type=self.rule_type) + } + self.create_posts(self.stream) + return self def __exit__(self, *args, **kwargs): pass def create_posts(self, stream): - pass + raise NotImplementedError def save(self): - pass + for post in self.instances: + post.save() class Meta: abstract = True diff --git a/src/newsreader/news/collection/feed.py b/src/newsreader/news/collection/feed.py index f67a109..cb5618f 100644 --- a/src/newsreader/news/collection/feed.py +++ b/src/newsreader/news/collection/feed.py @@ -37,20 +37,7 @@ logger = logging.getLogger(__name__) class FeedBuilder(Builder): - instances = [] - - def __enter__(self): - _, stream = self.stream - - self.instances = [] - self.existing_posts = { - post.remote_identifier: post - for post in Post.objects.filter( - rule=stream.rule, rule__type=RuleTypeChoices.feed - ) - } - - return super().__enter__() + rule__type = RuleTypeChoices.feed def create_posts(self, stream): data, stream = stream @@ -114,10 +101,6 @@ class FeedBuilder(Builder): content = "\n ".join([item.get("value") for item in items]) return self.sanitize_fragment(content) - def save(self): - for post in self.instances: - post.save() - class FeedStream(Stream): def read(self): diff --git a/src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py b/src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py new file mode 100644 index 0000000..2ce4cb3 --- /dev/null +++ b/src/newsreader/news/collection/migrations/0009_auto_20200807_2030.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.7 on 2020-08-07 18:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("collection", "0008_collectionrule_type")] + + operations = [ + migrations.AddField( + model_name="collectionrule", + name="screen_name", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name="collectionrule", + name="type", + field=models.CharField( + choices=[ + ("feed", "Feed"), + ("subreddit", "Subreddit"), + ("twitter", "Twitter"), + ], + default="feed", + max_length=20, + ), + ), + ] diff --git a/src/newsreader/news/collection/models.py b/src/newsreader/news/collection/models.py index 35841ba..5bada6a 100644 --- a/src/newsreader/news/collection/models.py +++ b/src/newsreader/news/collection/models.py @@ -57,11 +57,15 @@ class CollectionRule(TimeStampedModel): on_delete=models.CASCADE, ) + # Twitter + screen_name = models.CharField(max_length=255, blank=True, null=True) + objects = CollectionRuleQuerySet.as_manager() def __str__(self): return self.name + # TODO add twitter url @property def update_url(self): if self.type == RuleTypeChoices.subreddit: diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 557271c..7ef4784 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -94,18 +94,7 @@ def get_reddit_access_token(code, user): class RedditBuilder(Builder): - def __enter__(self): - _, stream = self.stream - - self.instances = [] - self.existing_posts = { - post.remote_identifier: post - for post in Post.objects.filter( - rule=stream.rule, rule__type=RuleTypeChoices.subreddit - ) - } - - return super().__enter__() + rule__type = RuleTypeChoices.subreddit def create_posts(self, stream): data, stream = stream @@ -218,10 +207,6 @@ class RedditBuilder(Builder): return results.values() - def save(self): - for post in self.instances: - post.save() - class RedditScheduler: max_amount = RATE_LIMIT diff --git a/src/newsreader/news/collection/tests/factories.py b/src/newsreader/news/collection/tests/factories.py index a84365d..761844b 100644 --- a/src/newsreader/news/collection/tests/factories.py +++ b/src/newsreader/news/collection/tests/factories.py @@ -32,3 +32,4 @@ class SubredditFactory(CollectionRuleFactory): class TwitterProfileFactory(CollectionRuleFactory): type = RuleTypeChoices.twitter + screen_name = factory.Faker("user_name") diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 8e79913..c27edb0 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -18,11 +18,10 @@ class TwitterBuilderTestCase(TestCase): def setUp(self): self.maxDiff = None - @skip("Not implemented") def test_simple_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory() + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) with builder((simple_mock, mock_stream)) as builder: @@ -39,10 +38,10 @@ class TwitterBuilderTestCase(TestCase): full_text = "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncatechars(full_text, 20)) + self.assertEquals(post.title, truncatechars(full_text, 40)) self.assertEquals(post.body, format_html(full_text)) - self.assertEquals(post.author, "Star Citizen") + self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291528756373286914" ) @@ -50,6 +49,22 @@ class TwitterBuilderTestCase(TestCase): post.publication_date, pytz.utc.localize(datetime(2020, 8, 7, 0, 17, 5)) ) + post = posts["1288550304095416320"] + + full_text = "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing." + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, truncatechars(full_text, 40)) + self.assertEquals(post.body, format_html(full_text)) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1288550304095416320" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 7, 29, 19, 1, 47)) + ) + # Note that only one media type can be uploaded to an Tweet # see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object @skip("Not implemented") diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 5061e38..2775841 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -1,4 +1,13 @@ +from datetime import datetime + +from django.template.defaultfilters import truncatechars +from django.utils.html import format_html + +import pytz + from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.core.models import Post TWITTER_URL = "https://twitter.com" @@ -9,18 +18,38 @@ class TwitterScheduler: class TwitterBuilder(Builder): - def __enter__(self): - _, stream = self.stream + rule__type = RuleTypeChoices.twitter - self.instances = [] - self.existing_posts = { - post.remote_identifier: post - for post in Post.objects.filter( - rule=stream.rule, rule__type=RuleTypeChoices.twitter + def create_posts(self, stream): + data, stream = stream + + if not data: + return + + self.instances = self.build(data, stream.rule) + + def build(self, posts, rule): + results = {} + + for post in posts: + remote_identifier = post["id_str"] + publication_date = pytz.utc.localize( + datetime.strptime(post["created_at"], "%a %b %d %H:%M:%S +0000 %Y") ) - } - return super().__enter__() + data = { + "remote_identifier": remote_identifier, + "title": truncatechars(post["full_text"], 40), + "body": format_html(post["full_text"]), + "author": rule.screen_name, + "publication_date": publication_date, + "url": (f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}"), + "rule": rule, + } + + results[remote_identifier] = Post(**data) + + return results.values() class TwitterStream(Stream): From 8f971a5c8963bcfabd5b8fdf8e5ae1ed25c7e914 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 7 Aug 2020 22:48:25 +0200 Subject: [PATCH 05/69] Add image mock with multiple images --- .../collection/tests/twitter/builder/mocks.py | 438 ++++-------------- src/newsreader/news/collection/twitter.py | 1 + 2 files changed, 101 insertions(+), 338 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index db7e410..bacaabb 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -204,405 +204,167 @@ image_mock = [ { "contributors": None, "coordinates": None, - "created_at": "Fri Aug 07 00:01:56 +0000 2020", - "display_text_range": [0, 278], + "created_at": "Fri Jun 05 22:51:46 +0000 2020", "entities": { "hashtags": [], "media": [ { - "display_url": "pic.twitter.com/aI5frhSKbI", - "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291524942970777601/photo/1", - "id": 1291524790956613633, - "id_str": "1291524790956613633", - "indices": [279, 302], - "media_url": "http://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", - "media_url_https": "https://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", + "display_url": "pic.twitter.com/VjEeDrL1iA", + "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1", + "id": 1269039233072689152, + "id_str": "1269039233072689152", + "indices": [2, 25], + "media_url": "http://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", + "media_url_https": "https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", "sizes": { - "large": {"h": 1080, "resize": "fit", "w": 1920}, - "medium": {"h": 675, "resize": "fit", "w": 1200}, - "small": {"h": 383, "resize": "fit", "w": 680}, + "large": {"h": 1073, "resize": "fit", "w": 1125}, + "medium": {"h": 1073, "resize": "fit", "w": 1125}, + "small": {"h": 649, "resize": "fit", "w": 680}, "thumb": {"h": 150, "resize": "crop", "w": 150}, }, "type": "photo", - "url": "https://t.co/aI5frhSKbI", + "url": "https://t.co/VjEeDrL1iA", } ], "symbols": [], - "urls": [ - { - "display_url": "youtu.be/rDy7tPf6CT8", - "expanded_url": "https://youtu.be/rDy7tPf6CT8", - "indices": [255, 278], - "url": "https://t.co/trAcIxBMlX", - } - ], + "urls": [], "user_mentions": [], }, "extended_entities": { "media": [ { - "display_url": "pic.twitter.com/aI5frhSKbI", - "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291524942970777601/photo/1", - "id": 1291524790956613633, - "id_str": "1291524790956613633", - "indices": [279, 302], - "media_url": "http://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", - "media_url_https": "https://pbs.twimg.com/media/Eexq9F-UYAERVd6.jpg", + "display_url": "pic.twitter.com/VjEeDrL1iA", + "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1", + "id": 1269039233072689152, + "id_str": "1269039233072689152", + "indices": [2, 25], + "media_url": "http://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", + "media_url_https": "https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg", "sizes": { - "large": {"h": 1080, "resize": "fit", "w": 1920}, - "medium": {"h": 675, "resize": "fit", "w": 1200}, - "small": {"h": 383, "resize": "fit", "w": 680}, + "large": {"h": 1073, "resize": "fit", "w": 1125}, + "medium": {"h": 1073, "resize": "fit", "w": 1125}, + "small": {"h": 649, "resize": "fit", "w": 680}, "thumb": {"h": 150, "resize": "crop", "w": 150}, }, "type": "photo", - "url": "https://t.co/aI5frhSKbI", - } + "url": "https://t.co/VjEeDrL1iA", + }, + { + "display_url": "pic.twitter.com/VjEeDrL1iA", + "expanded_url": "https://twitter.com/knxwledge/status/1269039237166321664/photo/1", + "id": 1269039233068527618, + "id_str": "1269039233068527618", + "indices": [2, 25], + "media_url": "http://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg", + "media_url_https": "https://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg", + "sizes": { + "large": {"h": 992, "resize": "fit", "w": 1472}, + "medium": {"h": 809, "resize": "fit", "w": 1200}, + "small": {"h": 458, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/VjEeDrL1iA", + }, ] }, - "favorite_count": 185, + "favorite_count": 2139, "favorited": False, - "full_text": "This week we strap into the cockpit and take a look at aviation experimentation in the Public Test Universe, before a Sprint Report affords looks at cargo decks, FPS weapons, day/night in the city, dynamic fire, and more. Also, penguiny hugs.\n\nWatch now: https://t.co/trAcIxBMlX https://t.co/aI5frhSKbI", "geo": None, - "id": 1291524942970777601, - "id_str": "1291524942970777601", + "id": 1269039237166321664, + "id_str": "1269039237166321664", "in_reply_to_screen_name": None, "in_reply_to_status_id": None, "in_reply_to_status_id_str": None, "in_reply_to_user_id": None, "in_reply_to_user_id_str": None, "is_quote_status": False, - "lang": "en", + "lang": "und", "place": None, "possibly_sensitive": False, - "retweet_count": 39, + "possibly_sensitive_appealable": False, + "retweet_count": 427, "retweeted": False, - "source": 'Twitter Web App', + "source": 'Twitter for iPhone', + "text": "_ https://t.co/VjEeDrL1iA", "truncated": False, "user": { "contributors_enabled": False, - "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "created_at": "Tue Nov 14 19:00:00 +0000 2017", "default_profile": False, "default_profile_image": False, - "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "description": "Grammy\u00ae Award Winning Beatmakr. https://t.co/SN23ei3EeC https://t.co/EkGRhZ1Bw9 https://t.co/eEb4NOmJLo", "entities": { - "description": {"urls": []}, + "description": { + "urls": [ + { + "display_url": "soundcloud.com/knxwledge", + "expanded_url": "http://soundcloud.com/knxwledge", + "indices": [32, 55], + "url": "https://t.co/SN23ei3EeC", + }, + { + "display_url": "knxwledge.bandcamp.com", + "expanded_url": "http://knxwledge.bandcamp.com", + "indices": [56, 79], + "url": "https://t.co/EkGRhZ1Bw9", + }, + { + "display_url": "twitch.tv/knxwledge", + "expanded_url": "http://twitch.tv/knxwledge", + "indices": [80, 103], + "url": "https://t.co/eEb4NOmJLo", + }, + ] + }, "url": { "urls": [ { - "display_url": "robertsspaceindustries.com", - "expanded_url": "http://www.robertsspaceindustries.com", + "display_url": "instagram.com/knxwledge/?hl=\u2026", + "expanded_url": "https://www.instagram.com/knxwledge/?hl=en", "indices": [0, 23], - "url": "https://t.co/iqO6apof3y", + "url": "https://t.co/UcMYfiQXLx", } ] }, }, - "favourites_count": 4588, + "favourites_count": 363, "follow_request_sent": None, - "followers_count": 106169, + "followers_count": 31194, "following": None, - "friends_count": 201, + "friends_count": 15, "geo_enabled": False, "has_extended_profile": False, - "id": 803542770, - "id_str": "803542770", + "id": 930510644763287552, + "id_str": "930510644763287552", "is_translation_enabled": False, "is_translator": False, "lang": None, - "listed_count": 890, - "location": "Roberts Space Industries", - "name": "Star Citizen", + "listed_count": 56, + "location": "", + "name": "knxwledge", "notifications": None, - "profile_background_color": "131516", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_color": "000000", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", - "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", - "profile_link_color": "0A5485", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "EFEFEF", - "profile_text_color": "333333", - "profile_use_background_image": True, + "profile_image_url": "http://pbs.twimg.com/profile_images/1274913160898592768/jFi4VDtJ_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1274913160898592768/jFi4VDtJ_normal.jpg", + "profile_link_color": "ABB8C2", + "profile_sidebar_border_color": "000000", + "profile_sidebar_fill_color": "000000", + "profile_text_color": "000000", + "profile_use_background_image": False, "protected": False, - "screen_name": "RobertsSpaceInd", - "statuses_count": 6210, + "screen_name": "knxwledge", + "statuses_count": 713, "time_zone": None, "translator_type": "none", - "url": "https://t.co/iqO6apof3y", + "url": "https://t.co/UcMYfiQXLx", "utc_offset": None, - "verified": True, + "verified": False, }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Thu Aug 06 00:03:04 +0000 2020", - "display_text_range": [0, 261], - "entities": { - "hashtags": [{"indices": [210, 222], "text": "StarCitizen"}], - "media": [ - { - "display_url": "pic.twitter.com/mHekGRycKa", - "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291162838791335936/photo/1", - "id": 1291160447526121473, - "id_str": "1291160447526121473", - "indices": [262, 285], - "media_url": "http://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", - "media_url_https": "https://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", - "sizes": { - "large": {"h": 1078, "resize": "fit", "w": 2048}, - "medium": {"h": 632, "resize": "fit", "w": 1200}, - "small": {"h": 358, "resize": "fit", "w": 680}, - "thumb": {"h": 150, "resize": "crop", "w": 150}, - }, - "type": "photo", - "url": "https://t.co/mHekGRycKa", - } - ], - "symbols": [], - "urls": [ - { - "display_url": "robertsspaceindustries.com/comm-link/tran\u2026", - "expanded_url": "https://robertsspaceindustries.com/comm-link/transmission/17712-Star-Citizen-Monthly-Report-July-2020", - "indices": [238, 261], - "url": "https://t.co/oVUjK6UWgC", - } - ], - "user_mentions": [], - }, - "extended_entities": { - "media": [ - { - "display_url": "pic.twitter.com/mHekGRycKa", - "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291162838791335936/photo/1", - "id": 1291160447526121473, - "id_str": "1291160447526121473", - "indices": [262, 285], - "media_url": "http://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", - "media_url_https": "https://pbs.twimg.com/media/Eesflg7UMAEW97M.jpg", - "sizes": { - "large": {"h": 1078, "resize": "fit", "w": 2048}, - "medium": {"h": 632, "resize": "fit", "w": 1200}, - "small": {"h": 358, "resize": "fit", "w": 680}, - "thumb": {"h": 150, "resize": "crop", "w": 150}, - }, - "type": "photo", - "url": "https://t.co/mHekGRycKa", - } - ] - }, - "favorite_count": 149, - "favorited": False, - "full_text": "We hope you\u2019re all enjoying the new additions that Alpha 3.10 brings to the \u2018verse. July\u2019s monthly report touches on a few of the final touches to the latest patch and work you can expect to see in the future. #StarCitizen\n\nRead it here: https://t.co/oVUjK6UWgC https://t.co/mHekGRycKa", - "geo": None, - "id": 1291162838791335936, - "id_str": "1291162838791335936", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 28, - "retweeted": False, - "source": 'Twitter Web App', - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed Sep 05 00:58:11 +0000 2012", - "default_profile": False, - "default_profile_image": False, - "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "robertsspaceindustries.com", - "expanded_url": "http://www.robertsspaceindustries.com", - "indices": [0, 23], - "url": "https://t.co/iqO6apof3y", - } - ] - }, - }, - "favourites_count": 4588, - "follow_request_sent": None, - "followers_count": 106169, - "following": None, - "friends_count": 201, - "geo_enabled": False, - "has_extended_profile": False, - "id": 803542770, - "id_str": "803542770", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 890, - "location": "Roberts Space Industries", - "name": "Star Citizen", - "notifications": None, - "profile_background_color": "131516", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", - "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", - "profile_link_color": "0A5485", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "EFEFEF", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "RobertsSpaceInd", - "statuses_count": 6210, - "time_zone": None, - "translator_type": "none", - "url": "https://t.co/iqO6apof3y", - "utc_offset": None, - "verified": True, - }, - }, - { - "contributors": None, - "coordinates": None, - "created_at": "Wed Aug 05 19:31:23 +0000 2020", - "display_text_range": [0, 218], - "entities": { - "hashtags": [], - "media": [ - { - "display_url": "pic.twitter.com/ESB5UBwhmO", - "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291094468473393152/photo/1", - "id": 1291090033080856576, - "id_str": "1291090033080856576", - "indices": [219, 242], - "media_url": "http://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", - "media_url_https": "https://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", - "sizes": { - "large": {"h": 635, "resize": "fit", "w": 1680}, - "medium": {"h": 454, "resize": "fit", "w": 1200}, - "small": {"h": 257, "resize": "fit", "w": 680}, - "thumb": {"h": 150, "resize": "crop", "w": 150}, - }, - "type": "photo", - "url": "https://t.co/ESB5UBwhmO", - } - ], - "symbols": [], - "urls": [ - { - "display_url": "robertsspaceindustries.com/spectrum/commu\u2026", - "expanded_url": "https://robertsspaceindustries.com/spectrum/community/SC/forum/3/thread/where-in-the-verse-is-pico-screenshot-contest", - "indices": [195, 218], - "url": "https://t.co/jFU7YLniDR", - } - ], - "user_mentions": [], - }, - "extended_entities": { - "media": [ - { - "display_url": "pic.twitter.com/ESB5UBwhmO", - "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291094468473393152/photo/1", - "id": 1291090033080856576, - "id_str": "1291090033080856576", - "indices": [219, 242], - "media_url": "http://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", - "media_url_https": "https://pbs.twimg.com/media/Eerfi2rU8AAej68.jpg", - "sizes": { - "large": {"h": 635, "resize": "fit", "w": 1680}, - "medium": {"h": 454, "resize": "fit", "w": 1200}, - "small": {"h": 257, "resize": "fit", "w": 680}, - "thumb": {"h": 150, "resize": "crop", "w": 150}, - }, - "type": "photo", - "url": "https://t.co/ESB5UBwhmO", - } - ] - }, - "favorite_count": 306, - "favorited": False, - "full_text": "Alpha 3.10 is live and our fuzzy friend Pico wants in on the action. \ud83d\udc27 We want to see what you and Pico are getting up to in the 'verse with our \"Where in the 'Verse is Pico?\" contest!\n\nDetails: https://t.co/jFU7YLniDR https://t.co/ESB5UBwhmO", - "geo": None, - "id": 1291094468473393152, - "id_str": "1291094468473393152", - "in_reply_to_screen_name": None, - "in_reply_to_status_id": None, - "in_reply_to_status_id_str": None, - "in_reply_to_user_id": None, - "in_reply_to_user_id_str": None, - "is_quote_status": False, - "lang": "en", - "place": None, - "possibly_sensitive": False, - "retweet_count": 65, - "retweeted": False, - "source": 'Twitter Web App', - "truncated": False, - "user": { - "contributors_enabled": False, - "created_at": "Wed Sep 05 00:58:11 +0000 2012", - "default_profile": False, - "default_profile_image": False, - "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", - "entities": { - "description": {"urls": []}, - "url": { - "urls": [ - { - "display_url": "robertsspaceindustries.com", - "expanded_url": "http://www.robertsspaceindustries.com", - "indices": [0, 23], - "url": "https://t.co/iqO6apof3y", - } - ] - }, - }, - "favourites_count": 4588, - "follow_request_sent": None, - "followers_count": 106169, - "following": None, - "friends_count": 201, - "geo_enabled": False, - "has_extended_profile": False, - "id": 803542770, - "id_str": "803542770", - "is_translation_enabled": False, - "is_translator": False, - "lang": None, - "listed_count": 890, - "location": "Roberts Space Industries", - "name": "Star Citizen", - "notifications": None, - "profile_background_color": "131516", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", - "profile_background_tile": False, - "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", - "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", - "profile_link_color": "0A5485", - "profile_sidebar_border_color": "FFFFFF", - "profile_sidebar_fill_color": "EFEFEF", - "profile_text_color": "333333", - "profile_use_background_image": True, - "protected": False, - "screen_name": "RobertsSpaceInd", - "statuses_count": 6210, - "time_zone": None, - "translator_type": "none", - "url": "https://t.co/iqO6apof3y", - "utc_offset": None, - "verified": True, - }, - }, + } ] video_mock = [ diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 2775841..a33fc84 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -11,6 +11,7 @@ from newsreader.news.core.models import Post TWITTER_URL = "https://twitter.com" +TWITTER_API_URL = "https://api.twitter.com/1.1" class TwitterScheduler: From 9fd3a845e7399c6614451d15635ef8284d96bef6 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 8 Aug 2020 15:58:44 +0200 Subject: [PATCH 06/69] Add image test --- src/newsreader/news/collection/choices.py | 6 +++ .../collection/tests/twitter/builder/mocks.py | 2 +- .../collection/tests/twitter/builder/tests.py | 43 +++++++++++++++++-- src/newsreader/news/collection/twitter.py | 20 ++++++++- 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/newsreader/news/collection/choices.py b/src/newsreader/news/collection/choices.py index 47e304e..8a129de 100644 --- a/src/newsreader/news/collection/choices.py +++ b/src/newsreader/news/collection/choices.py @@ -6,3 +6,9 @@ class RuleTypeChoices(TextChoices): feed = "feed", _("Feed") subreddit = "subreddit", _("Subreddit") twitter = "twitter", _("Twitter") + + +class TwitterPostTypeChoices(TextChoices): + photo = "photo", _("Poto") + video = "video", _("Video") + animated_gif = "animated_gif", _("GIF") diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index bacaabb..6c17fef 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -286,7 +286,7 @@ image_mock = [ "retweet_count": 427, "retweeted": False, "source": 'Twitter for iPhone', - "text": "_ https://t.co/VjEeDrL1iA", + "full_text": "_ https://t.co/VjEeDrL1iA", "truncated": False, "user": { "contributors_enabled": False, diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index c27edb0..e941a36 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -9,7 +9,10 @@ from django.utils.html import format_html import pytz from newsreader.news.collection.tests.factories import TwitterProfileFactory -from newsreader.news.collection.tests.twitter.builder.mocks import simple_mock +from newsreader.news.collection.tests.twitter.builder.mocks import ( + image_mock, + simple_mock, +) from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder from newsreader.news.core.models import Post @@ -67,9 +70,43 @@ class TwitterBuilderTestCase(TestCase): # Note that only one media type can be uploaded to an Tweet # see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object - @skip("Not implemented") def test_images_in_post(self): - pass + builder = TwitterBuilder + + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder((image_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual(("1269039237166321664",), posts.keys()) + + post = posts["1269039237166321664"] + + full_text = "_ https://t.co/VjEeDrL1iA" + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, full_text) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1269039237166321664" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 6, 5, 22, 51, 46)) + ) + + self.assertIn(full_text, post.body) + self.assertIn( + f"
1269039233072689152
", + post.body, + ) + self.assertIn( + f"
1269039233068527618
", + post.body, + ) @skip("Not implemented") def test_videos_in_post(self): diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index a33fc84..76e2457 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -6,7 +6,7 @@ from django.utils.html import format_html import pytz from newsreader.news.collection.base import Builder, Client, Collector, Stream -from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices from newsreader.news.core.models import Post @@ -37,11 +37,27 @@ class TwitterBuilder(Builder): publication_date = pytz.utc.localize( datetime.strptime(post["created_at"], "%a %b %d %H:%M:%S +0000 %Y") ) + body = "" + + if "extended_entities" in post: + media_entities = post["extended_entities"]["media"] + + for media_entity in media_entities: + media_type = media_entity["type"] + media_url = media_entity["media_url_https"] + title = media_entity["id_str"] + + if media_type == TwitterPostTypeChoices.photo: + html_fragment = f"
{title}
" + + body += format_html(html_fragment) + + body += format_html(post["full_text"]) data = { "remote_identifier": remote_identifier, "title": truncatechars(post["full_text"], 40), - "body": format_html(post["full_text"]), + "body": body, "author": rule.screen_name, "publication_date": publication_date, "url": (f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}"), From bafb18863693f44f0a2d8ff74ea8b39ec8626fd2 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 9 Aug 2020 22:43:39 +0200 Subject: [PATCH 07/69] WIP: video posts --- .../collection/tests/twitter/builder/tests.py | 44 +++++++++++++++++-- src/newsreader/news/collection/twitter.py | 36 ++++++++++++++- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index e941a36..f7ef547 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -12,6 +12,7 @@ from newsreader.news.collection.tests.factories import TwitterProfileFactory from newsreader.news.collection.tests.twitter.builder.mocks import ( image_mock, simple_mock, + video_mock, ) from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder from newsreader.news.core.models import Post @@ -68,7 +69,7 @@ class TwitterBuilderTestCase(TestCase): post.publication_date, pytz.utc.localize(datetime(2020, 7, 29, 19, 1, 47)) ) - # Note that only one media type can be uploaded to an Tweet + # note that only one media type can be uploaded to an Tweet # see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object def test_images_in_post(self): builder = TwitterBuilder @@ -108,9 +109,46 @@ class TwitterBuilderTestCase(TestCase): post.body, ) - @skip("Not implemented") def test_videos_in_post(self): - pass + builder = TwitterBuilder + + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder((video_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291080532361527296", "1291079386821582849"), posts.keys() + ) + + post = posts["1291080532361527296"] + + full_text = ( + "Small enough to access hard-to-reach ore deposits, but with enough" + "power to get through the tough jobs, Greycat\u2019s ROC perfectly" + "complements any mining operation. \n\nDetails:" + "https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH" + ) + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, truncatechars(full_text, 40)) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291080532361527296" + ) + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 8, 5, 18, 36, 0)) + ) + + self.assertIn(full_text, post.body) + self.assertIn( + """
""", + post.body, + ) @skip("Not implemented") def test_GIFs_in_post(self): diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 76e2457..a45493b 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -48,9 +48,41 @@ class TwitterBuilder(Builder): title = media_entity["id_str"] if media_type == TwitterPostTypeChoices.photo: - html_fragment = f"
{title}
" + html_fragment = format_html( + "
{title}
", + title=title, + media_url=media_url, + ) - body += format_html(html_fragment) + body += html_fragment + + elif media_type == TwitterPostTypeChoices.video: + meta_data = media_entity["video_info"] + + # TODO catch case where bitrates are not defined or no videos + video = next( + iter( + sorted( + ( + video + for video in meta_data["variants"] + if "bitrate" in video + ), + reverse=True, + key=lambda video: video["bitrate"], + ) + ) + ) + + content_type = video["content_type"] + url = video["url"] + + html_fragment = format_html( + """
""", + url=url, + content_type=content_type, + ) + body += html_fragment body += format_html(post["full_text"]) From 9bdbc2a47ed85b2c06da50871def4a902fa71a35 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 4 Sep 2020 23:45:24 +0200 Subject: [PATCH 08/69] Add ftfy for fixing surrogates --- poetry.lock | 576 ++++++++++++++++++++++++++----------------------- pyproject.toml | 1 + 2 files changed, 303 insertions(+), 274 deletions(-) diff --git a/poetry.lock b/poetry.lock index cab45d1..9dd9d1b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,40 +1,40 @@ [[package]] -category = "main" -description = "Low-level AMQP client for Python (fork of amqplib)." name = "amqp" +version = "2.5.2" +description = "Low-level AMQP client for Python (fork of amqplib)." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.5.2" [package.dependencies] vine = ">=1.1.3,<5.0.0a1" [[package]] -category = "dev" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." name = "appdirs" +version = "1.4.3" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = "*" -version = "1.4.3" [[package]] -category = "main" -description = "ASGI specs, helper code, and adapters" name = "asgiref" +version = "3.2.7" +description = "ASGI specs, helper code, and adapters" +category = "main" optional = false python-versions = ">=3.5" -version = "3.2.7" [package.extras] tests = ["pytest (>=4.3.0,<4.4.0)", "pytest-asyncio (>=0.10.0,<0.11.0)"] [[package]] -category = "dev" -description = "Classes Without Boilerplate" name = "attrs" +version = "19.3.0" +description = "Classes Without Boilerplate" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "19.3.0" [package.extras] azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] @@ -43,46 +43,49 @@ docs = ["sphinx", "zope.interface"] tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] [[package]] -category = "dev" -description = "Removes unused imports and unused variables" name = "autoflake" +version = "1.3.1" +description = "Removes unused imports and unused variables" +category = "dev" optional = false python-versions = "*" -version = "1.3.1" [package.dependencies] pyflakes = ">=1.1.0" [[package]] -category = "main" -description = "Screen-scraping library" name = "beautifulsoup4" +version = "4.9.0" +description = "Screen-scraping library" +category = "main" optional = false python-versions = "*" -version = "4.9.0" - -[package.dependencies] -soupsieve = [">1.2", "<2.0"] [package.extras] html5lib = ["html5lib"] lxml = ["lxml"] -[[package]] -category = "main" -description = "Python multiprocessing fork with improvements and bugfixes" -name = "billiard" -optional = false -python-versions = "*" -version = "3.6.3.0" +[package.dependencies] +soupsieve = [">1.2", "<2.0"] + +[[package]] +name = "billiard" +version = "3.6.3.0" +description = "Python multiprocessing fork with improvements and bugfixes" +category = "main" +optional = false +python-versions = "*" [[package]] -category = "dev" -description = "The uncompromising code formatter." name = "black" +version = "19.3b0" +description = "The uncompromising code formatter." +category = "dev" optional = false python-versions = ">=3.6" -version = "19.3b0" + +[package.extras] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [package.dependencies] appdirs = "*" @@ -90,34 +93,25 @@ attrs = ">=18.1.0" click = ">=6.5" toml = ">=0.9.4" -[package.extras] -d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] - [[package]] -category = "main" -description = "An easy safelist-based HTML-sanitizing tool." name = "bleach" +version = "3.1.4" +description = "An easy safelist-based HTML-sanitizing tool." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "3.1.4" [package.dependencies] six = ">=1.9.0" webencodings = "*" [[package]] -category = "main" -description = "Distributed Task Queue." name = "celery" +version = "4.4.2" +description = "Distributed Task Queue." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*," -version = "4.4.2" - -[package.dependencies] -billiard = ">=3.6.3.0,<4.0" -kombu = ">=4.6.8,<4.7" -pytz = ">0.0-dev" -vine = "1.3.0" [package.extras] arangodb = ["pyArango (>=1.3.2)"] @@ -153,37 +147,43 @@ yaml = ["PyYAML (>=3.10)"] zookeeper = ["kazoo (>=1.3.1)"] zstd = ["zstandard"] +[package.dependencies] +billiard = ">=3.6.3.0,<4.0" +kombu = ">=4.6.8,<4.7" +pytz = ">0.0-dev" +vine = "1.3.0" + [[package]] -category = "main" -description = "Python package for providing Mozilla's CA Bundle." name = "certifi" -optional = false -python-versions = "*" version = "2020.4.5.1" - -[[package]] +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "chardet" +version = "3.0.4" +description = "Universal encoding detector for Python 2 and 3" category = "main" -description = "Universal encoding detector for Python 2 and 3" -name = "chardet" optional = false python-versions = "*" -version = "3.0.4" [[package]] -category = "dev" -description = "Composable command line interface toolkit" name = "click" +version = "7.1.1" +description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "7.1.1" [[package]] -category = "main" -description = "Python client library for Core API." name = "coreapi" +version = "2.3.3" +description = "Python client library for Core API." +category = "main" optional = false python-versions = "*" -version = "2.3.3" [package.dependencies] coreschema = "*" @@ -192,62 +192,62 @@ requests = "*" uritemplate = "*" [[package]] -category = "main" -description = "Core Schema." name = "coreschema" +version = "0.0.4" +description = "Core Schema." +category = "main" optional = false python-versions = "*" -version = "0.0.4" [package.dependencies] jinja2 = "*" [[package]] -category = "dev" -description = "Code coverage measurement for Python" name = "coverage" +version = "5.1" +description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "5.1" [package.extras] toml = ["toml"] [[package]] -category = "main" -description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." name = "django" +version = "3.0.7" +description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." +category = "main" optional = false python-versions = ">=3.6" -version = "3.0.7" + +[package.extras] +argon2 = ["argon2-cffi (>=16.1.0)"] +bcrypt = ["bcrypt"] [package.dependencies] asgiref = ">=3.2,<4.0" pytz = "*" sqlparse = ">=0.2.2" -[package.extras] -argon2 = ["argon2-cffi (>=16.1.0)"] -bcrypt = ["bcrypt"] - [[package]] -category = "main" -description = "A helper class for handling configuration defaults of packaged apps gracefully." name = "django-appconf" +version = "1.0.4" +description = "A helper class for handling configuration defaults of packaged apps gracefully." +category = "main" optional = false python-versions = "*" -version = "1.0.4" [package.dependencies] django = "*" [[package]] -category = "main" -description = "Keep track of failed login attempts in Django-powered sites." name = "django-axes" +version = "5.3.1" +description = "Keep track of failed login attempts in Django-powered sites." +category = "main" optional = false python-versions = "~=3.6" -version = "5.3.1" [package.dependencies] django = ">=1.11" @@ -255,93 +255,96 @@ django-appconf = ">=1.0.3" django-ipware = ">=2.0.2" [[package]] -category = "main" -description = "Database-backed Periodic Tasks." name = "django-celery-beat" +version = "2.0.0" +description = "Database-backed Periodic Tasks." +category = "main" optional = false python-versions = "*" -version = "2.0.0" [package.dependencies] -Django = ">=1.11.17" celery = "*" +Django = ">=1.11.17" django-timezone-field = ">=4.0,<5.0" python-crontab = ">=2.3.4" [[package]] -category = "dev" -description = "A configurable set of panels that display various debug information about the current request/response." name = "django-debug-toolbar" +version = "2.2" +description = "A configurable set of panels that display various debug information about the current request/response." +category = "dev" optional = false python-versions = ">=3.5" -version = "2.2" [package.dependencies] Django = ">=1.11" sqlparse = ">=0.2.0" [[package]] -category = "dev" -description = "Extensions for Django" name = "django-extensions" +version = "2.2.9" +description = "Extensions for Django" +category = "dev" optional = false python-versions = "*" -version = "2.2.9" [package.dependencies] six = ">=1.2" [[package]] -category = "main" -description = "A Django utility application that returns client's real IP address" name = "django-ipware" -optional = false -python-versions = "*" version = "2.1.0" - -[[package]] +description = "A Django utility application that returns client's real IP address" category = "main" -description = "An extensible user-registration application for Django" -name = "django-registration-redux" optional = false python-versions = "*" -version = "2.7" [[package]] +name = "django-registration-redux" +version = "2.7" +description = "An extensible user-registration application for Django" category = "main" -description = "A Django app providing database and form fields for pytz timezone objects." +optional = false +python-versions = "*" + +[[package]] name = "django-timezone-field" +version = "4.0" +description = "A Django app providing database and form fields for pytz timezone objects." +category = "main" optional = false python-versions = ">=3.5" -version = "4.0" [package.dependencies] django = ">=2.2" pytz = "*" [[package]] -category = "main" -description = "Web APIs for Django, made easy." name = "djangorestframework" +version = "3.11.0" +description = "Web APIs for Django, made easy." +category = "main" optional = false python-versions = ">=3.5" -version = "3.11.0" [package.dependencies] django = ">=1.11" [[package]] -category = "main" -description = "Automated generation of real Swagger/OpenAPI 2.0 schemas from Django Rest Framework code." name = "drf-yasg" +version = "1.17.1" +description = "Automated generation of real Swagger/OpenAPI 2.0 schemas from Django Rest Framework code." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "1.17.1" + +[package.extras] +validation = ["swagger-spec-validator (>=2.1.0)"] [package.dependencies] -Django = ">=1.11.7" coreapi = ">=2.3.3" coreschema = ">=0.0.4" +Django = ">=1.11.7" djangorestframework = ">=3.8" inflection = ">=0.3.1" packaging = "*" @@ -349,62 +352,67 @@ packaging = "*" six = ">=1.10.0" uritemplate = ">=3.0.0" -[package.extras] -validation = ["swagger-spec-validator (>=2.1.0)"] - [[package]] -category = "dev" -description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." name = "factory-boy" +version = "2.12.0" +description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.12.0" [package.dependencies] Faker = ">=0.7.0" [[package]] -category = "dev" -description = "Faker is a Python package that generates fake data for you." name = "faker" +version = "4.0.2" +description = "Faker is a Python package that generates fake data for you." +category = "dev" optional = false python-versions = ">=3.4" -version = "4.0.2" [package.dependencies] python-dateutil = ">=2.4" text-unidecode = "1.3" [[package]] -category = "main" -description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" name = "feedparser" +version = "5.2.1" +description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" +category = "main" optional = false python-versions = "*" -version = "5.2.1" [[package]] -category = "dev" -description = "Let your Python tests travel through time" name = "freezegun" +version = "0.3.15" +description = "Let your Python tests travel through time" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.3.15" [package.dependencies] python-dateutil = ">=1.0,<2.0 || >2.0" six = "*" [[package]] +name = "ftfy" +version = "5.8" +description = "Fixes some problems with Unicode text after the fact" category = "main" -description = "WSGI HTTP Server for UNIX" -name = "gunicorn" optional = false -python-versions = ">=3.4" -version = "20.0.4" +python-versions = ">=3.5" [package.dependencies] -setuptools = ">=3.0" +wcwidth = "*" + +[[package]] +name = "gunicorn" +version = "20.0.4" +description = "WSGI HTTP Server for UNIX" +category = "main" +optional = false +python-versions = ">=3.4" [package.extras] eventlet = ["eventlet (>=0.9.7)"] @@ -412,45 +420,48 @@ gevent = ["gevent (>=0.13)"] setproctitle = ["setproctitle"] tornado = ["tornado (>=0.2)"] +[package.dependencies] +setuptools = ">=3.0" + [[package]] -category = "main" -description = "Internationalized Domain Names in Applications (IDNA)" name = "idna" +version = "2.9" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.9" [[package]] -category = "main" -description = "Read metadata from Python packages" -marker = "python_version < \"3.8\"" name = "importlib-metadata" +version = "1.6.0" +description = "Read metadata from Python packages" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.6.0" - -[package.dependencies] -zipp = ">=0.5" +marker = "python_version < \"3.8\"" [package.extras] docs = ["sphinx", "rst.linker"] testing = ["packaging", "importlib-resources"] -[[package]] -category = "main" -description = "A port of Ruby on Rails inflector to Python" -name = "inflection" -optional = false -python-versions = ">=3.5" -version = "0.4.0" +[package.dependencies] +zipp = ">=0.5" + +[[package]] +name = "inflection" +version = "0.4.0" +description = "A port of Ruby on Rails inflector to Python" +category = "main" +optional = false +python-versions = ">=3.5" [[package]] -category = "dev" -description = "A Python utility / library to sort Python imports." name = "isort" +version = "4.3.21" +description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "4.3.21" [package.extras] pipfile = ["pipreqs", "requirementslib"] @@ -459,41 +470,34 @@ requirements = ["pipreqs", "pip-api"] xdg_home = ["appdirs (>=1.4.0)"] [[package]] -category = "main" -description = "Simple immutable types for python." name = "itypes" +version = "1.1.0" +description = "Simple immutable types for python." +category = "main" optional = false python-versions = "*" -version = "1.1.0" [[package]] -category = "main" -description = "A very fast and expressive template engine." name = "jinja2" +version = "2.11.1" +description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.11.1" - -[package.dependencies] -MarkupSafe = ">=0.23" [package.extras] i18n = ["Babel (>=0.8)"] +[package.dependencies] +MarkupSafe = ">=0.23" + [[package]] -category = "main" -description = "Messaging library for Python." name = "kombu" +version = "4.6.8" +description = "Messaging library for Python." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "4.6.8" - -[package.dependencies] -amqp = ">=2.5.2,<2.6" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.18" [package.extras] azureservicebus = ["azure-servicebus (>=0.21.1)"] @@ -511,13 +515,20 @@ sqs = ["boto3 (>=1.4.4)", "pycurl (7.43.0.2)"] yaml = ["PyYAML (>=3.10)"] zookeeper = ["kazoo (>=1.3.1)"] +[package.dependencies] +amqp = ">=2.5.2,<2.6" + +[package.dependencies.importlib-metadata] +version = ">=0.18" +python = "<3.8" + [[package]] -category = "main" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." name = "lxml" +version = "4.5.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" -version = "4.5.0" [package.extras] cssselect = ["cssselect (>=0.7)"] @@ -526,112 +537,116 @@ htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] [[package]] -category = "main" -description = "Safely add untrusted strings to HTML/XML markup." name = "markupsafe" +version = "1.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -version = "1.1.1" [[package]] -category = "main" -description = "Core utilities for Python packages" name = "packaging" +version = "20.3" +description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "20.3" [package.dependencies] pyparsing = ">=2.0.2" six = "*" [[package]] -category = "main" -description = "psycopg2 - Python-PostgreSQL Database Adapter" name = "psycopg2-binary" +version = "2.8.5" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -version = "2.8.5" [[package]] -category = "dev" -description = "passive checker of Python programs" name = "pyflakes" +version = "2.2.0" +description = "passive checker of Python programs" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.2.0" [[package]] -category = "main" -description = "Python parsing module" name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "2.4.7" [[package]] -category = "main" -description = "Python Crontab API" name = "python-crontab" +version = "2.4.1" +description = "Python Crontab API" +category = "main" optional = false python-versions = "*" -version = "2.4.1" - -[package.dependencies] -python-dateutil = "*" [package.extras] cron-description = ["cron-descriptor"] cron-schedule = ["croniter"] +[package.dependencies] +python-dateutil = "*" + [[package]] -category = "main" -description = "Extensions to the standard Python datetime module" name = "python-dateutil" +version = "2.8.1" +description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -version = "2.8.1" [package.dependencies] six = ">=1.5" [[package]] -category = "main" -description = "Add .env support to your django/flask apps in development and deployments" name = "python-dotenv" +version = "0.12.0" +description = "Add .env support to your django/flask apps in development and deployments" +category = "main" optional = false python-versions = "*" -version = "0.12.0" [package.extras] cli = ["click (>=5.0)"] [[package]] -category = "main" -description = "Pure python memcached client" name = "python-memcached" +version = "1.59" +description = "Pure python memcached client" +category = "main" optional = false python-versions = "*" -version = "1.59" [package.dependencies] six = ">=1.4.0" [[package]] -category = "main" -description = "World timezone definitions, modern and historical" name = "pytz" +version = "2019.3" +description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" -version = "2019.3" [[package]] -category = "main" -description = "Python HTTP for Humans." name = "requests" +version = "2.23.0" +description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.23.0" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [package.dependencies] certifi = ">=2017.4.17" @@ -639,47 +654,39 @@ chardet = ">=3.0.2,<4" idna = ">=2.5,<3" urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" -[package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] - [[package]] -category = "main" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" name = "ruamel.yaml" +version = "0.16.10" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +category = "main" optional = false python-versions = "*" -version = "0.16.10" - -[package.dependencies] -[package.dependencies."ruamel.yaml.clib"] -python = "<3.9" -version = ">=0.1.2" [package.extras] docs = ["ryd"] jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] -[[package]] -category = "main" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -marker = "platform_python_implementation == \"CPython\" and python_version < \"3.9\"" -name = "ruamel.yaml.clib" -optional = false -python-versions = "*" -version = "0.2.0" - -[[package]] -category = "main" -description = "Python client for Sentry (https://getsentry.com)" -name = "sentry-sdk" -optional = false -python-versions = "*" -version = "0.15.1" - [package.dependencies] -certifi = "*" -urllib3 = ">=1.10.0" +[package.dependencies."ruamel.yaml.clib"] +version = ">=0.1.2" +python = "<3.9" + +[[package]] +name = "ruamel.yaml.clib" +version = "0.2.0" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = "*" +marker = "platform_python_implementation == \"CPython\" and python_version < \"3.9\"" + +[[package]] +name = "sentry-sdk" +version = "0.15.1" +description = "Python client for Sentry (https://getsentry.com)" +category = "main" +optional = false +python-versions = "*" [package.extras] aiohttp = ["aiohttp (>=3.5)"] @@ -695,69 +702,73 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] tornado = ["tornado (>=5)"] +[package.dependencies] +certifi = "*" +urllib3 = ">=1.10.0" + [[package]] -category = "main" -description = "Python 2 and 3 compatibility utilities" name = "six" +version = "1.14.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.14.0" [[package]] -category = "main" -description = "A modern CSS selector implementation for Beautiful Soup." name = "soupsieve" +version = "1.9.5" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" optional = false python-versions = "*" -version = "1.9.5" [[package]] -category = "main" -description = "Non-validating SQL parser" name = "sqlparse" +version = "0.3.1" +description = "Non-validating SQL parser" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.3.1" [[package]] -category = "dev" -description = "Traceback serialization library." name = "tblib" +version = "1.6.0" +description = "Traceback serialization library." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "1.6.0" [[package]] -category = "dev" -description = "The most basic Text::Unidecode port" name = "text-unidecode" -optional = false -python-versions = "*" version = "1.3" - -[[package]] +description = "The most basic Text::Unidecode port" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "toml" +version = "0.10.0" +description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" -description = "Python Library for Tom's Obvious, Minimal Language" -name = "toml" optional = false python-versions = "*" -version = "0.10.0" [[package]] -category = "main" -description = "URI templates" name = "uritemplate" +version = "3.0.1" +description = "URI templates" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "3.0.1" [[package]] -category = "main" -description = "HTTP library with thread-safe connection pooling, file post, and more." name = "urllib3" +version = "1.25.8" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "1.25.8" [package.extras] brotli = ["brotlipy (>=0.6.0)"] @@ -765,37 +776,46 @@ secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "cer socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] -category = "main" -description = "Promises, promises, promises." name = "vine" +version = "1.3.0" +description = "Promises, promises, promises." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.3.0" [[package]] +name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" category = "main" -description = "Character encoding aliases for legacy web content" -name = "webencodings" optional = false python-versions = "*" -version = "0.5.1" [[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" category = "main" -description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" +optional = false +python-versions = "*" + +[[package]] name = "zipp" +version = "3.1.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.6" -version = "3.1.0" +marker = "python_version < \"3.8\"" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "6b207d452b10de2399c4c49118da997dda6ed1bb0437963c3f415ecd3d806fe5" +lock-version = "1.0" python-versions = "^3.7" +content-hash = "c1083804e80713b99e49109f59560ebfaeba67c4a903a15130a9748c23922b1a" [metadata.files] amqp = [ @@ -951,6 +971,9 @@ freezegun = [ {file = "freezegun-0.3.15-py2.py3-none-any.whl", hash = "sha256:82c757a05b7c7ca3e176bfebd7d6779fd9139c7cb4ef969c38a28d74deef89b2"}, {file = "freezegun-0.3.15.tar.gz", hash = "sha256:e2062f2c7f95cc276a834c22f1a17179467176b624cc6f936e8bc3be5535ad1b"}, ] +ftfy = [ + {file = "ftfy-5.8.tar.gz", hash = "sha256:51c7767f8c4b47d291fcef30b9625fb5341c06a31e6a3b627039c706c42f3720"}, +] gunicorn = [ {file = "gunicorn-20.0.4-py2.py3-none-any.whl", hash = "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"}, {file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"}, @@ -1110,6 +1133,7 @@ pytz = [ {file = "pytz-2019.3.tar.gz", hash = "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"}, ] requests = [ + {file = "requests-2.23.0-py2.7.egg", hash = "sha256:5d2d0ffbb515f39417009a46c14256291061ac01ba8f875b90cad137de83beb4"}, {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, ] @@ -1179,6 +1203,10 @@ vine = [ {file = "vine-1.3.0-py2.py3-none-any.whl", hash = "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"}, {file = "vine-1.3.0.tar.gz", hash = "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87"}, ] +wcwidth = [ + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, +] webencodings = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, diff --git a/pyproject.toml b/pyproject.toml index bdc34a9..b235671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ gunicorn = "^20.0.4" python-dotenv = "^0.12.0" django = ">=3.0.7" sentry-sdk = "^0.15.1" +ftfy = "^5.8" [tool.poetry.dev-dependencies] factory-boy = "^2.12.0" From 30ed1c13f939ee141a99f99a614a703262cea984 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 6 Sep 2020 20:14:41 +0200 Subject: [PATCH 09/69] Fix video post tests/code --- .../collection/tests/twitter/builder/mocks.py | 153 ++++++++++++++++++ .../collection/tests/twitter/builder/tests.py | 43 ++++- src/newsreader/news/collection/twitter.py | 31 ++-- 3 files changed, 203 insertions(+), 24 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index 6c17fef..364d762 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -700,6 +700,159 @@ video_mock = [ }, ] +video_without_bitrate_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Wed Aug 05 18:36:00 +0000 2020", + "display_text_range": [0, 196], + "entities": { + "hashtags": [], + "media": [ + { + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/mZ8CAuq3SH", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "robertsspaceindustries.com/greycatroc", + "expanded_url": "http://robertsspaceindustries.com/greycatroc", + "indices": [173, 196], + "url": "https://t.co/2aH7qdOfSk", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "additional_media_info": { + "description": "", + "embeddable": True, + "monetizable": False, + "title": "", + }, + "display_url": "pic.twitter.com/mZ8CAuq3SH", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1291080532361527296/video/1", + "id": 1291074294747770880, + "id_str": "1291074294747770880", + "indices": [197, 220], + "media_url": "http://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "media_url_https": "https://pbs.twimg.com/media/EerWyexUEAQhRL1.jpg", + "sizes": { + "large": {"h": 720, "resize": "fit", "w": 1280}, + "medium": {"h": 675, "resize": "fit", "w": 1200}, + "small": {"h": 383, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "video", + "url": "https://t.co/mZ8CAuq3SH", + "video_info": { + "aspect_ratio": [16, 9], + "duration_millis": 82967, + "variants": [ + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/amplify_video/1291074294747770880/pl/kMYgFEoRyoW99o-i.m3u8?tag=13", + } + ], + }, + } + ] + }, + "favorite_count": 289, + "favorited": False, + "full_text": "Small enough to access hard-to-reach ore deposits, but with enough power to get through the tough jobs, Greycat\u2019s ROC perfectly complements any mining operation. \n\nDetails: https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH", + "geo": None, + "id": 1291080532361527296, + "id_str": "1291080532361527296", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 64, + "retweeted": False, + "source": 'Twitter Media Studio', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + } +] + retweet_mock = [ { "contributors": None, diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index f7ef547..bf66648 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -8,11 +8,14 @@ from django.utils.html import format_html import pytz +from ftfy import fix_text + from newsreader.news.collection.tests.factories import TwitterProfileFactory from newsreader.news.collection.tests.twitter.builder.mocks import ( image_mock, simple_mock, video_mock, + video_without_bitrate_mock, ) from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder from newsreader.news.core.models import Post @@ -100,11 +103,11 @@ class TwitterBuilderTestCase(TestCase): ) self.assertIn(full_text, post.body) - self.assertIn( + self.assertInHTML( f"
1269039233072689152
", post.body, ) - self.assertIn( + self.assertInHTML( f"
1269039233068527618
", post.body, ) @@ -126,11 +129,11 @@ class TwitterBuilderTestCase(TestCase): post = posts["1291080532361527296"] - full_text = ( + full_text = fix_text( "Small enough to access hard-to-reach ore deposits, but with enough" - "power to get through the tough jobs, Greycat\u2019s ROC perfectly" - "complements any mining operation. \n\nDetails:" - "https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH" + " power to get through the tough jobs, Greycat\u2019s ROC perfectly" + " complements any mining operation. \n\nDetails:" + " https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH" ) self.assertEquals(post.rule, profile) @@ -145,9 +148,31 @@ class TwitterBuilderTestCase(TestCase): ) self.assertIn(full_text, post.body) - self.assertIn( + self.assertInHTML( """
""", post.body, + count=1, + ) + + def test_video_without_bitrate(self): + builder = TwitterBuilder + + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder((video_without_bitrate_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual(("1291080532361527296",), posts.keys()) + + post = posts["1291080532361527296"] + + self.assertInHTML( + """
""", + post.body, + count=1, ) @skip("Not implemented") @@ -189,3 +214,7 @@ class TwitterBuilderTestCase(TestCase): @skip("Not implemented") def test_duplicate_in_data(self): pass + + @skip("Not implemented") + def test_invalid_unicode_chars(self): + pass diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index a45493b..8fbf331 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -5,6 +5,8 @@ from django.utils.html import format_html import pytz +from ftfy import fix_text + from newsreader.news.collection.base import Builder, Client, Collector, Stream from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices from newsreader.news.core.models import Post @@ -59,26 +61,21 @@ class TwitterBuilder(Builder): elif media_type == TwitterPostTypeChoices.video: meta_data = media_entity["video_info"] - # TODO catch case where bitrates are not defined or no videos - video = next( - iter( - sorted( - ( - video - for video in meta_data["variants"] - if "bitrate" in video - ), - reverse=True, - key=lambda video: video["bitrate"], - ) - ) + videos = sorted( + [video for video in meta_data["variants"]], + reverse=True, + key=lambda video: video.get("bitrate", 0), ) + if not videos: + continue + + video = videos[0] content_type = video["content_type"] url = video["url"] html_fragment = format_html( - """
""", + """
""", url=url, content_type=content_type, ) @@ -88,11 +85,11 @@ class TwitterBuilder(Builder): data = { "remote_identifier": remote_identifier, - "title": truncatechars(post["full_text"], 40), - "body": body, + "title": fix_text(truncatechars(post["full_text"], 40)), + "body": fix_text(body), "author": rule.screen_name, "publication_date": publication_date, - "url": (f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}"), + "url": f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}", "rule": rule, } From d2dcd0fa7eab8b781af1f47a89dffcaca2fc4770 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Wed, 9 Sep 2020 21:49:48 +0200 Subject: [PATCH 10/69] Add animated_gif test scenario --- .../collection/tests/twitter/builder/mocks.py | 11 ++++++++ .../collection/tests/twitter/builder/tests.py | 26 +++++++++++++++++-- src/newsreader/news/collection/twitter.py | 5 +++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index 364d762..b114531 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -1,5 +1,9 @@ # retrieved with: # curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=twitterapi&tweet_mode=extended" | python3 -m json.tool --sort-keys +# +# see https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/tweet-object +# and https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/extended-entities-object +# for more information about tweet objects simple_mock = [ { @@ -200,6 +204,8 @@ simple_mock = [ }, ] +# contains tweets with "extended_entities" keys which contains native media objects +# which are in this case of the type "photo" image_mock = [ { "contributors": None, @@ -367,6 +373,8 @@ image_mock = [ } ] +# contains tweets with "extended_entities" keys which contains native media objects +# which are in this case of the type "video" video_mock = [ { "contributors": None, @@ -853,6 +861,7 @@ video_without_bitrate_mock = [ } ] +# contains tweets with "retweeted_status" keys containing the retweeted tweet retweet_mock = [ { "contributors": None, @@ -1774,6 +1783,8 @@ quoted_mock = [ }, ] +# contains tweets with "extended_entities" keys which contains native media objects +# which are in this case of the type "animated_gif" gif_mock = [ { "contributors": None, diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index bf66648..2b21a3b 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -12,6 +12,7 @@ from ftfy import fix_text from newsreader.news.collection.tests.factories import TwitterProfileFactory from newsreader.news.collection.tests.twitter.builder.mocks import ( + gif_mock, image_mock, simple_mock, video_mock, @@ -175,9 +176,30 @@ class TwitterBuilderTestCase(TestCase): count=1, ) - @skip("Not implemented") def test_GIFs_in_post(self): - pass + builder = TwitterBuilder + + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder((gif_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1289337776140296193", "1288965215648849920"), posts.keys() + ) + + post = posts["1289337776140296193"] + + self.assertInHTML( + """
""", + post.body, + count=1, + ) + + self.assertIn("@Xenosystems https://t.co/wxvioLCJ6h", post.body) @skip("Not implemented") def test_retweet_post(self): diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 8fbf331..7b48da2 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -58,7 +58,10 @@ class TwitterBuilder(Builder): body += html_fragment - elif media_type == TwitterPostTypeChoices.video: + elif media_type in ( + TwitterPostTypeChoices.video, + TwitterPostTypeChoices.animated_gif, + ): meta_data = media_entity["video_info"] videos = sorted( From cfe9c29a14efee4eb91d797e075592b9b42038f1 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 12 Sep 2020 16:29:36 +0200 Subject: [PATCH 11/69] Add tests for retweeted tweets & quoted tweets --- .../collection/tests/twitter/builder/mocks.py | 5 +- .../collection/tests/twitter/builder/tests.py | 73 +++++++++++++++---- src/newsreader/news/collection/twitter.py | 7 ++ 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index b114531..11047b1 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -861,7 +861,8 @@ video_without_bitrate_mock = [ } ] -# contains tweets with "retweeted_status" keys containing the retweeted tweet +# contains tweets with a "retweeted_status" key containing the retweeted tweet. +# the "retweet" cannot add hashtags, URLs or other details, see https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/entities-object#retweets-quote retweet_mock = [ { "contributors": None, @@ -1280,6 +1281,8 @@ retweet_mock = [ }, ] +# contains tweets with a "quoted_status" key containing the quoted tweet. +# quoted tweets can add hashtags, URL's and other details as it adds content "on top" of the quoted tweet see https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/overview/entities-object#retweets-quotes quoted_mock = [ { "contributors": None, diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 2b21a3b..2d7150e 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -14,6 +14,8 @@ from newsreader.news.collection.tests.factories import TwitterProfileFactory from newsreader.news.collection.tests.twitter.builder.mocks import ( gif_mock, image_mock, + quoted_mock, + retweet_mock, simple_mock, video_mock, video_without_bitrate_mock, @@ -201,25 +203,70 @@ class TwitterBuilderTestCase(TestCase): self.assertIn("@Xenosystems https://t.co/wxvioLCJ6h", post.body) - @skip("Not implemented") def test_retweet_post(self): - pass + builder = TwitterBuilder + + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder((retweet_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291117030486106112", "1288825524878336000"), posts.keys() + ) + + post = posts["1291117030486106112"] + + self.assertIn( + fix_text( + "RT @Narayan_N7: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo," + " the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPle\u2026" + ), + post.body, + ) + + self.assertIn( + fix_text( + "Original tweet: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch" + " 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease," + " share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe!" + " \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\nhttps://t.co/j4QahHzbw4" + ), + post.body, + ) - @skip("Not implemented") def test_quoted_post(self): - pass + builder = TwitterBuilder - @skip("Not implemented") - def test_images_in_quoted_post(self): - pass + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) - @skip("Not implemented") - def test_videos_in_quoted_post(self): - pass + with builder((quoted_mock, mock_stream)) as builder: + builder.save() - @skip("Not implemented") - def test_GIFs_in_quoted_post(self): - pass + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1290801039075979264", "1289320160021495809"), posts.keys() + ) + + post = posts["1290801039075979264"] + + self.assertIn( + fix_text("Bonne nuit \ud83c\udf3a\ud83d\udeeb https://t.co/WyznJwCJLp"), + post.body, + ) + + self.assertIn( + fix_text( + "Quoted tweet: #Starcitizen Le jeu est beau. Bonne nuit" + " @RobertsSpaceInd https://t.co/xCXun68V3r" + ), + post.body, + ) @skip("Not implemented") def test_empty_data(self): diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 7b48da2..ae29047 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -84,6 +84,13 @@ class TwitterBuilder(Builder): ) body += html_fragment + if "retweeted_status" in post: + original_post = post["retweeted_status"] + body += format_html(f"Original tweet: {original_post['full_text']}") + if "quoted_status" in post: + original_post = post["quoted_status"] + body += format_html(f"Quoted tweet: {original_post['full_text']}") + body += format_html(post["full_text"]) data = { From a90e55865505fb18d65cddda7059be14a1889383 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 12 Sep 2020 16:49:01 +0200 Subject: [PATCH 12/69] Move media_entities parsing to seperate function --- .../collection/tests/twitter/builder/tests.py | 6 +- src/newsreader/news/collection/twitter.py | 114 ++++++++++-------- 2 files changed, 71 insertions(+), 49 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 2d7150e..b3561d8 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -107,12 +107,14 @@ class TwitterBuilderTestCase(TestCase): self.assertIn(full_text, post.body) self.assertInHTML( - f"
1269039233072689152
", + """
1269039233072689152
""", post.body, + count=1, ) self.assertInHTML( - f"
1269039233068527618
", + """
1269039233068527618
""", post.body, + count=1, ) def test_videos_in_post(self): diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index ae29047..d8d273b 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -1,3 +1,5 @@ +import logging + from datetime import datetime from django.template.defaultfilters import truncatechars @@ -12,6 +14,8 @@ from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeC from newsreader.news.core.models import Post +logger = logging.getLogger(__name__) + TWITTER_URL = "https://twitter.com" TWITTER_API_URL = "https://api.twitter.com/1.1" @@ -36,62 +40,31 @@ class TwitterBuilder(Builder): for post in posts: remote_identifier = post["id_str"] + url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" publication_date = pytz.utc.localize( datetime.strptime(post["created_at"], "%a %b %d %H:%M:%S +0000 %Y") ) - body = "" + body = post["full_text"] if "extended_entities" in post: - media_entities = post["extended_entities"]["media"] - - for media_entity in media_entities: - media_type = media_entity["type"] - media_url = media_entity["media_url_https"] - title = media_entity["id_str"] - - if media_type == TwitterPostTypeChoices.photo: - html_fragment = format_html( - "
{title}
", - title=title, - media_url=media_url, - ) - - body += html_fragment - - elif media_type in ( - TwitterPostTypeChoices.video, - TwitterPostTypeChoices.animated_gif, - ): - meta_data = media_entity["video_info"] - - videos = sorted( - [video for video in meta_data["variants"]], - reverse=True, - key=lambda video: video.get("bitrate", 0), - ) - - if not videos: - continue - - video = videos[0] - content_type = video["content_type"] - url = video["url"] - - html_fragment = format_html( - """
""", - url=url, - content_type=content_type, - ) - body += html_fragment + try: + media_entities = self.get_media_entities(post) + body += media_entities + except KeyError: + logger.exception(f"Failed parsing media_entities for {url}") if "retweeted_status" in post: original_post = post["retweeted_status"] - body += format_html(f"Original tweet: {original_post['full_text']}") + body += format_html( + "Original tweet: {original_post}", + original_post=original_post["full_text"], + ) if "quoted_status" in post: original_post = post["quoted_status"] - body += format_html(f"Quoted tweet: {original_post['full_text']}") - - body += format_html(post["full_text"]) + body += format_html( + "Quoted tweet: {original_post}", + original_post=original_post["full_text"], + ) data = { "remote_identifier": remote_identifier, @@ -99,7 +72,7 @@ class TwitterBuilder(Builder): "body": fix_text(body), "author": rule.screen_name, "publication_date": publication_date, - "url": f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}", + "url": url, "rule": rule, } @@ -107,6 +80,53 @@ class TwitterBuilder(Builder): return results.values() + def get_media_entities(self, post): + media_entities = post["extended_entities"]["media"] + formatted_entities = "" + + for media_entity in media_entities: + media_type = media_entity["type"] + media_url = media_entity["media_url_https"] + title = media_entity["id_str"] + + if media_type == TwitterPostTypeChoices.photo: + html_fragment = format_html( + """
{title}
""", + title=title, + media_url=media_url, + ) + + formatted_entities += html_fragment + + elif media_type in ( + TwitterPostTypeChoices.video, + TwitterPostTypeChoices.animated_gif, + ): + meta_data = media_entity["video_info"] + + videos = sorted( + [video for video in meta_data["variants"]], + reverse=True, + key=lambda video: video.get("bitrate", 0), + ) + + if not videos: + continue + + video = videos[0] + content_type = video["content_type"] + url = video["url"] + + html_fragment = format_html( + """
""", + url=url, + content_type=content_type, + ) + + formatted_entities += html_fragment + + return formatted_entities + class TwitterStream(Stream): pass From b48efbbffbf874da4fbe32edd98a87cb99cbc5d3 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 12 Sep 2020 17:08:03 +0200 Subject: [PATCH 13/69] Remove invalid unicode text test as this is tested already in the video post mock --- src/newsreader/news/collection/tests/twitter/builder/tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index b3561d8..9ba81d4 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -285,7 +285,3 @@ class TwitterBuilderTestCase(TestCase): @skip("Not implemented") def test_duplicate_in_data(self): pass - - @skip("Not implemented") - def test_invalid_unicode_chars(self): - pass From 6be8862a7d79dbce406c307042094e14cc686b07 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 12 Sep 2020 19:23:59 +0200 Subject: [PATCH 14/69] Add last builder test --- src/newsreader/news/collection/base.py | 18 +++ src/newsreader/news/collection/constants.py | 5 +- src/newsreader/news/collection/feed.py | 17 --- src/newsreader/news/collection/reddit.py | 13 +-- .../collection/tests/twitter/builder/mocks.py | 105 ++++++++++++++++++ .../collection/tests/twitter/builder/tests.py | 65 +++++++---- src/newsreader/news/collection/twitter.py | 11 +- 7 files changed, 180 insertions(+), 54 deletions(-) diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index 5de3454..df122a6 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -1,5 +1,11 @@ +import bleach + from bs4 import BeautifulSoup +from newsreader.news.collection.constants import ( + WHITELISTED_ATTRIBUTES, + WHITELISTED_TAGS, +) from newsreader.news.collection.exceptions import StreamParseException from newsreader.news.collection.utils import fetch from newsreader.news.core.models import Post @@ -79,6 +85,18 @@ class Builder: def create_posts(self, stream): raise NotImplementedError + def sanitize_fragment(self, fragment): + if not fragment: + return "" + + return bleach.clean( + fragment, + tags=WHITELISTED_TAGS, + attributes=WHITELISTED_ATTRIBUTES, + strip=True, + strip_comments=True, + ) + def save(self): for post in self.instances: post.save() diff --git a/src/newsreader/news/collection/constants.py b/src/newsreader/news/collection/constants.py index eade898..0c73642 100644 --- a/src/newsreader/news/collection/constants.py +++ b/src/newsreader/news/collection/constants.py @@ -23,6 +23,7 @@ WHITELISTED_TAGS = ( WHITELISTED_ATTRIBUTES = { **BLEACH_ATTRIBUTES, "a": ["href", "rel"], - "img": ["alt", "src"], - "source": ["srcset", "media", "src", "type"], + "img": ["alt", "src", "loading"], + "video": ["controls", "muted"], + "source": ["srcset", "src", "media", "type"], } diff --git a/src/newsreader/news/collection/feed.py b/src/newsreader/news/collection/feed.py index cb5618f..ff28666 100644 --- a/src/newsreader/news/collection/feed.py +++ b/src/newsreader/news/collection/feed.py @@ -6,17 +6,12 @@ from datetime import timedelta from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.utils import timezone -import bleach import pytz from feedparser import parse from newsreader.news.collection.base import Builder, Client, Collector, Stream from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.constants import ( - WHITELISTED_ATTRIBUTES, - WHITELISTED_TAGS, -) from newsreader.news.collection.exceptions import ( StreamDeniedException, StreamException, @@ -85,18 +80,6 @@ class FeedBuilder(Builder): yield Post(**data) - def sanitize_fragment(self, fragment): - if not fragment: - return "" - - return bleach.clean( - fragment, - tags=WHITELISTED_TAGS, - attributes=WHITELISTED_ATTRIBUTES, - strip=True, - strip_comments=True, - ) - def get_content(self, items): content = "\n ".join([item.get("value") for item in items]) return self.sanitize_fragment(content) diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 7ef4784..65ce384 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -12,7 +12,6 @@ from django.core.cache import cache from django.utils import timezone from django.utils.html import format_html -import bleach import pytz import requests @@ -128,17 +127,7 @@ class RedditBuilder(Builder): if is_text_post: uncleaned_body = data["selftext_html"] unescaped_body = unescape(uncleaned_body) if uncleaned_body else "" - body = ( - bleach.clean( - unescaped_body, - tags=WHITELISTED_TAGS, - attributes=WHITELISTED_ATTRIBUTES, - strip=True, - strip_comments=True, - ) - if unescaped_body - else "" - ) + body = self.sanitize_fragment(unescaped_body) if unescaped_body else "" elif direct_url.endswith(REDDIT_IMAGE_EXTENSIONS): body = format_html( "
{title}
", diff --git a/src/newsreader/news/collection/tests/twitter/builder/mocks.py b/src/newsreader/news/collection/tests/twitter/builder/mocks.py index 11047b1..b330f2f 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/builder/mocks.py @@ -2080,3 +2080,108 @@ gif_mock = [ }, }, ] + +unsanitized_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Aug 07 00:17:05 +0000 2020", + "display_text_range": [11, 59], + "entities": { + "hashtags": [], + "symbols": [], + "urls": [ + { + "display_url": "youtu.be/rDy7tPf6CT8", + "expanded_url": "https://youtu.be/rDy7tPf6CT8", + "indices": [36, 59], + "url": "https://t.co/trAcIxBMlX", + } + ], + "user_mentions": [ + { + "id": 975844884606275587, + "id_str": "975844884606275587", + "indices": [0, 10], + "name": "ArieNeo", + "screen_name": "ArieNeoSC", + } + ], + }, + "favorite_count": 19, + "favorited": False, + "full_text": "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX
", + "geo": None, + "id": 1291528756373286914, + "id_str": "1291528756373286914", + "in_reply_to_screen_name": "ArieNeoSC", + "in_reply_to_status_id": 1291507356313038850, + "in_reply_to_status_id_str": "1291507356313038850", + "in_reply_to_user_id": 975844884606275587, + "in_reply_to_user_id_str": "975844884606275587", + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 5, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4588, + "follow_request_sent": None, + "followers_count": 106169, + "following": None, + "friends_count": 201, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 890, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6210, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + } +] diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 9ba81d4..19fdce3 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -1,8 +1,6 @@ from datetime import datetime -from unittest import skip from unittest.mock import MagicMock -from django.template.defaultfilters import truncatechars from django.test import TestCase from django.utils.html import format_html @@ -17,10 +15,12 @@ from newsreader.news.collection.tests.twitter.builder.mocks import ( quoted_mock, retweet_mock, simple_mock, + unsanitized_mock, video_mock, video_without_bitrate_mock, ) from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder +from newsreader.news.collection.utils import truncate_text from newsreader.news.core.models import Post @@ -48,7 +48,7 @@ class TwitterBuilderTestCase(TestCase): full_text = "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncatechars(full_text, 40)) + self.assertEquals(post.title, truncate_text(Post, "title", full_text)) self.assertEquals(post.body, format_html(full_text)) self.assertEquals(post.author, "RobertsSpaceInd") @@ -64,7 +64,7 @@ class TwitterBuilderTestCase(TestCase): full_text = "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing." self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncatechars(full_text, 40)) + self.assertEquals(post.title, truncate_text(Post, "title", full_text)) self.assertEquals(post.body, format_html(full_text)) self.assertEquals(post.author, "RobertsSpaceInd") @@ -107,12 +107,12 @@ class TwitterBuilderTestCase(TestCase): self.assertIn(full_text, post.body) self.assertInHTML( - """
1269039233072689152
""", + """
1269039233072689152
""", post.body, count=1, ) self.assertInHTML( - """
1269039233068527618
""", + """
1269039233068527618
""", post.body, count=1, ) @@ -142,7 +142,7 @@ class TwitterBuilderTestCase(TestCase): ) self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncatechars(full_text, 40)) + self.assertEquals(post.title, truncate_text(Post, "title", full_text)) self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( @@ -154,7 +154,7 @@ class TwitterBuilderTestCase(TestCase): self.assertIn(full_text, post.body) self.assertInHTML( - """
""", + """
""", post.body, count=1, ) @@ -175,7 +175,7 @@ class TwitterBuilderTestCase(TestCase): post = posts["1291080532361527296"] self.assertInHTML( - """
""", + """
""", post.body, count=1, ) @@ -198,7 +198,7 @@ class TwitterBuilderTestCase(TestCase): post = posts["1289337776140296193"] self.assertInHTML( - """
""", + """
""", post.body, count=1, ) @@ -270,18 +270,43 @@ class TwitterBuilderTestCase(TestCase): post.body, ) - @skip("Not implemented") def test_empty_data(self): - pass + builder = TwitterBuilder - @skip("Not implemented") - def test_update_posts(self): - pass + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder(([], mock_stream)) as builder: + builder.save() + + self.assertEquals(Post.objects.count(), 0) - @skip("Not implemented") def test_html_sanitizing(self): - pass + builder = TwitterBuilder - @skip("Not implemented") - def test_duplicate_in_data(self): - pass + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = MagicMock(rule=profile) + + with builder((unsanitized_mock, mock_stream)) as builder: + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual(("1291528756373286914",), posts.keys()) + + post = posts["1291528756373286914"] + + full_text = ( + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" + "
" + ) + + self.assertEquals(post.rule, profile) + self.assertEquals(post.title, truncate_text(Post, "title", full_text)) + self.assertEquals(post.body, format_html(full_text)) + + self.assertInHTML("", post.body, count=0) + self.assertInHTML("
", post.body, count=1) + + self.assertInHTML("", post.title, count=0) + self.assertInHTML("
", post.title, count=1) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index d8d273b..b0f08cc 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -2,7 +2,6 @@ import logging from datetime import datetime -from django.template.defaultfilters import truncatechars from django.utils.html import format_html import pytz @@ -11,6 +10,7 @@ from ftfy import fix_text from newsreader.news.collection.base import Builder, Client, Collector, Stream from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices +from newsreader.news.collection.utils import truncate_text from newsreader.news.core.models import Post @@ -41,10 +41,13 @@ class TwitterBuilder(Builder): for post in posts: remote_identifier = post["id_str"] url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" + + body = post["full_text"] + title = truncate_text(Post, "title", self.sanitize_fragment(body)) + publication_date = pytz.utc.localize( datetime.strptime(post["created_at"], "%a %b %d %H:%M:%S +0000 %Y") ) - body = post["full_text"] if "extended_entities" in post: try: @@ -66,9 +69,11 @@ class TwitterBuilder(Builder): original_post=original_post["full_text"], ) + body = self.sanitize_fragment(body) + data = { "remote_identifier": remote_identifier, - "title": fix_text(truncatechars(post["full_text"], 40)), + "title": fix_text(title), "body": fix_text(body), "author": rule.screen_name, "publication_date": publication_date, From 150c49262895adef3613638e57eaf4fd3751fbcf Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 13 Sep 2020 13:32:52 +0200 Subject: [PATCH 15/69] Alot of plumbing --- src/newsreader/news/collection/base.py | 107 +++++++------- src/newsreader/news/collection/choices.py | 2 +- src/newsreader/news/collection/favicon.py | 93 +++++++----- src/newsreader/news/collection/feed.py | 90 +++++------- src/newsreader/news/collection/reddit.py | 133 +++++++++--------- .../collection/tests/favicon/builder/tests.py | 32 ++++- .../collection/tests/favicon/client/tests.py | 28 ++-- .../tests/favicon/collector/tests.py | 23 +-- .../collection/tests/feed/builder/tests.py | 82 ++++++----- .../collection/tests/feed/client/tests.py | 4 +- .../collection/tests/feed/collector/tests.py | 4 +- .../collection/tests/feed/stream/tests.py | 6 +- .../collection/tests/reddit/builder/tests.py | 82 ++++++----- src/newsreader/news/collection/tests/tests.py | 96 +++++++------ .../collection/tests/twitter/builder/tests.py | 27 ++-- src/newsreader/news/collection/twitter.py | 40 +++--- 16 files changed, 462 insertions(+), 387 deletions(-) diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index df122a6..700fae5 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -1,13 +1,10 @@ import bleach -from bs4 import BeautifulSoup - from newsreader.news.collection.constants import ( WHITELISTED_ATTRIBUTES, WHITELISTED_TAGS, ) -from newsreader.news.collection.exceptions import StreamParseException -from newsreader.news.collection.utils import fetch +from newsreader.news.collection.models import CollectionRule from newsreader.news.core.models import Post @@ -33,7 +30,7 @@ class Stream: class Client: """ - Retrieves the data with streams + Retrieves the data through streams """ stream = Stream @@ -56,33 +53,24 @@ class Client: class Builder: """ - Creates the collected posts + Builds instances of various types """ instances = [] stream = None - rule_type = None + payload = None - def __init__(self, stream): + def __init__(self, payload, stream): + self.payload = payload self.stream = stream def __enter__(self): - _, stream = self.stream - - self.instances = [] - self.existing_posts = { - post.remote_identifier: post - for post in Post.objects.filter(rule=stream.rule, rule__type=self.rule_type) - } - - self.create_posts(self.stream) - return self def __exit__(self, *args, **kwargs): pass - def create_posts(self, stream): + def build(self): raise NotImplementedError def sanitize_fragment(self, fragment): @@ -97,10 +85,6 @@ class Builder: strip_comments=True, ) - def save(self): - for post in self.instances: - post.save() - class Meta: abstract = True @@ -118,46 +102,59 @@ class Collector: self.builder = builder if builder else self.builder def collect(self, rules=None): - with self.client(rules=rules) as client: - for data, stream in client: - with self.builder((data, stream)) as builder: - builder.save() + raise NotImplementedError class Meta: abstract = True -class WebsiteStream(Stream): - def __init__(self, url): - self.url = url +class PostBuilder(Builder): + rule_type = None - def read(self): - response = fetch(self.url) - - return (self.parse(response.content), self) - - def parse(self, payload): - try: - return BeautifulSoup(payload, "lxml") - except TypeError: - raise StreamParseException("Could not parse given HTML") - - -class URLBuilder(Builder): def __enter__(self): - return self + self.existing_posts = { + post.remote_identifier: post + for post in Post.objects.filter( + rule=self.stream.rule, rule__type=self.rule_type + ) + } - def build(self): - data, stream = self.stream - rule = stream.rule + return super().__enter__() - try: - url = data["feed"]["link"] - except (KeyError, TypeError): - url = None + def save(self): + for post in self.instances: + post.save() - if url: - rule.website_url = url - rule.save() + class Meta: + abstract = True - return rule, url + +class PostStream(Stream): + rule_type = None + + +class PostClient(Client): + stream = PostStream + + def __init__(self, rules=[]): + if rules: + self.rules = rules + else: + self.rules = CollectionRule.objects.enabled().filter( + type=self.stream.rule_type + ) + + def set_rule_error(self, rule, exception): + length = rule._meta.get_field("error").max_length + + rule.error = exception.message[-length:] + rule.succeeded = False + + +class PostCollector(Collector): + def collect(self, rules=None): + with self.client(rules=rules) as client: + for payload, stream in client: + with self.builder(payload, stream) as builder: + builder.build() + builder.save() diff --git a/src/newsreader/news/collection/choices.py b/src/newsreader/news/collection/choices.py index 8a129de..3fd9bef 100644 --- a/src/newsreader/news/collection/choices.py +++ b/src/newsreader/news/collection/choices.py @@ -9,6 +9,6 @@ class RuleTypeChoices(TextChoices): class TwitterPostTypeChoices(TextChoices): - photo = "photo", _("Poto") + photo = "photo", _("Photo") video = "video", _("Video") animated_gif = "animated_gif", _("GIF") diff --git a/src/newsreader/news/collection/favicon.py b/src/newsreader/news/collection/favicon.py index 44b96bf..639e7f6 100644 --- a/src/newsreader/news/collection/favicon.py +++ b/src/newsreader/news/collection/favicon.py @@ -1,16 +1,12 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import urljoin, urlparse -from newsreader.news.collection.base import ( - Builder, - Client, - Collector, - Stream, - URLBuilder, - WebsiteStream, -) -from newsreader.news.collection.exceptions import StreamException +from bs4 import BeautifulSoup + +from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.exceptions import StreamException, StreamParseException from newsreader.news.collection.feed import FeedClient +from newsreader.news.collection.utils import fetch LINK_RELS = [ @@ -21,17 +17,45 @@ LINK_RELS = [ ] +class WebsiteStream(Stream): + def read(self): + response = fetch(self.rule.website_url) + + return self.parse(response.content), self + + def parse(self, payload): + try: + return BeautifulSoup(payload, features="lxml") + except TypeError: + raise StreamParseException("Could not parse given HTML") + + +class WebsiteURLBuilder(Builder): + def build(self): + try: + url = self.payload["feed"]["link"] + except (KeyError, TypeError): + url = None + + self.instances = [(self.stream, url)] if url else [] + + def save(self): + for stream, url in self.instances: + stream.rule.website_url = url + stream.rule.save() + + class FaviconBuilder(Builder): def build(self): - rule, soup = self.stream + rule = self.stream.rule - url = self.parse(soup, rule.website_url) + url = self.parse() - if url: - rule.favicon = url - rule.save() + self.instances = [(rule, url)] if url else [] + + def parse(self): + soup = self.payload - def parse(self, soup, website_url): if not soup.head: return @@ -44,9 +68,9 @@ class FaviconBuilder(Builder): parsed_url = urlparse(url) if not parsed_url.scheme and not parsed_url.netloc: - if not website_url: + if not self.stream.rule.website_url: return - return urljoin(website_url, url) + return urljoin(self.stream.rule.website_url, url) elif not parsed_url.scheme: return urljoin(f"https://{parsed_url.netloc}", parsed_url.path) @@ -73,6 +97,11 @@ class FaviconBuilder(Builder): elif icons: return icons.pop() + def save(self): + for rule, favicon_url in self.instances: + rule.favicon = favicon_url + rule.save() + class FaviconClient(Client): stream = WebsiteStream @@ -82,39 +111,35 @@ class FaviconClient(Client): def __enter__(self): with ThreadPoolExecutor(max_workers=10) as executor: - futures = { - executor.submit(stream.read): rule for rule, stream in self.streams - } + futures = [executor.submit(stream.read) for stream in self.streams] for future in as_completed(futures): - rule = futures[future] - try: - response_data, stream = future.result() + payload, stream = future.result() except StreamException: continue - yield (rule, response_data) + yield payload, stream class FaviconCollector(Collector): feed_client, favicon_client = (FeedClient, FaviconClient) - url_builder, favicon_builder = (URLBuilder, FaviconBuilder) + url_builder, favicon_builder = (WebsiteURLBuilder, FaviconBuilder) def collect(self, rules=None): streams = [] with self.feed_client(rules=rules) as client: - for data, stream in client: - with self.url_builder((data, stream)) as builder: - rule, url = builder.build() + for payload, stream in client: + with self.url_builder(payload, stream) as builder: + builder.build() + builder.save() - if not url: - continue - - streams.append((rule, WebsiteStream(url))) + if builder.instances: + streams.append(WebsiteStream(stream.rule)) with self.favicon_client(streams) as client: - for rule, data in client: - with self.favicon_builder((rule, data)) as builder: + for payload, stream in client: + with self.favicon_builder(payload, stream) as builder: builder.build() + builder.save() diff --git a/src/newsreader/news/collection/feed.py b/src/newsreader/news/collection/feed.py index ff28666..22f8dc7 100644 --- a/src/newsreader/news/collection/feed.py +++ b/src/newsreader/news/collection/feed.py @@ -10,7 +10,12 @@ import pytz from feedparser import parse -from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.base import ( + PostBuilder, + PostClient, + PostCollector, + PostStream, +) from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.exceptions import ( StreamDeniedException, @@ -19,7 +24,6 @@ from newsreader.news.collection.exceptions import ( StreamParseException, StreamTimeOutException, ) -from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.utils import ( build_publication_date, fetch, @@ -31,19 +35,10 @@ from newsreader.news.core.models import Post logger = logging.getLogger(__name__) -class FeedBuilder(Builder): +class FeedBuilder(PostBuilder): rule__type = RuleTypeChoices.feed - def create_posts(self, stream): - data, stream = stream - - with FeedDuplicateHandler(stream.rule) as duplicate_handler: - entries = data.get("entries", []) - - instances = self.build(entries, stream.rule) - self.instances = duplicate_handler.check(instances) - - def build(self, entries, rule): + def build(self): field_mapping = { "id": "remote_identifier", "title": "title", @@ -52,40 +47,47 @@ class FeedBuilder(Builder): "published_parsed": "publication_date", "author": "author", } + tz = pytz.timezone(self.stream.rule.timezone) + instances = [] - tz = pytz.timezone(rule.timezone) + with FeedDuplicateHandler(self.stream.rule) as duplicate_handler: + entries = self.payload.get("entries", []) - for entry in entries: - data = {"rule_id": rule.pk} + for entry in entries: + data = {"rule_id": self.stream.rule.pk} - for field, model_field in field_mapping.items(): - if not field in entry: - continue + for field, model_field in field_mapping.items(): + if not field in entry: + continue - value = truncate_text(Post, model_field, entry[field]) + value = truncate_text(Post, model_field, entry[field]) - if field == "published_parsed": - data[model_field] = build_publication_date(value, tz) - elif field == "summary": - data[model_field] = self.sanitize_fragment(value) - else: - data[model_field] = value + if field == "published_parsed": + data[model_field] = build_publication_date(value, tz) + elif field == "summary": + data[model_field] = self.sanitize_fragment(value) + else: + data[model_field] = value - if "content" in entry: - content = self.get_content(entry["content"]) - body = data.get("body", "") + if "content" in entry: + content = self.get_content(entry["content"]) + body = data.get("body", "") - if not body or len(body) < len(content): - data["body"] = content + if not body or len(body) < len(content): + data["body"] = content - yield Post(**data) + instances.append(Post(**data)) + + self.instances = duplicate_handler.check(instances) def get_content(self, items): content = "\n ".join([item.get("value") for item in items]) return self.sanitize_fragment(content) -class FeedStream(Stream): +class FeedStream(PostStream): + rule_type = RuleTypeChoices.feed + def read(self): response = fetch(self.rule.url) @@ -99,17 +101,9 @@ class FeedStream(Stream): raise StreamParseException(response=response, message=message) from e -class FeedClient(Client): +class FeedClient(PostClient): stream = FeedStream - def __init__(self, rules=[]): - if rules: - self.rules = rules - else: - self.rules = CollectionRule.objects.filter( - enabled=True, type=RuleTypeChoices.feed - ) - def __enter__(self): streams = [self.stream(rule) for rule in self.rules] @@ -120,13 +114,13 @@ class FeedClient(Client): stream = futures[future] try: - response_data = future.result() + payload = future.result() stream.rule.error = None stream.rule.succeeded = True stream.rule.last_suceeded = timezone.now() - yield response_data + yield payload except (StreamNotFoundException, StreamTimeOutException) as e: logger.warning(f"Request failed for {stream.rule.url}") @@ -142,14 +136,8 @@ class FeedClient(Client): finally: stream.rule.save() - def set_rule_error(self, rule, exception): - length = rule._meta.get_field("error").max_length - rule.error = exception.message[-length:] - rule.succeeded = False - - -class FeedCollector(Collector): +class FeedCollector(PostCollector): builder = FeedBuilder client = FeedClient diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 65ce384..82a4d96 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -15,7 +15,12 @@ from django.utils.html import format_html import pytz import requests -from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.base import ( + PostBuilder, + PostClient, + PostCollector, + PostStream, +) from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.constants import ( WHITELISTED_ATTRIBUTES, @@ -92,21 +97,17 @@ def get_reddit_access_token(code, user): return response_data["access_token"], response_data["refresh_token"] -class RedditBuilder(Builder): - rule__type = RuleTypeChoices.subreddit +class RedditBuilder(PostBuilder): + rule_type = RuleTypeChoices.subreddit - def create_posts(self, stream): - data, stream = stream - posts = [] + def build(self): + results = {} - if not "data" in data or not "children" in data["data"]: + if not "data" in self.payload or not "children" in self.payload["data"]: return - posts = data["data"]["children"] - self.instances = self.build(posts, stream.rule) - - def build(self, posts, rule): - results = {} + posts = self.payload["data"]["children"] + rule = self.stream.rule for post in posts: if not "data" in post or post["kind"] != REDDIT_POST: @@ -170,7 +171,9 @@ class RedditBuilder(Builder): parsed_date = datetime.fromtimestamp(post["data"]["created_utc"]) created_date = pytz.utc.localize(parsed_date) except (OverflowError, OSError): - logging.warning(f"Failed parsing timestamp from {url_fragment}") + logging.warning( + f"Failed parsing timestamp from {REDDIT_URL}{post_url_fragment}" + ) created_date = timezone.now() post_data = { @@ -194,52 +197,11 @@ class RedditBuilder(Builder): results[remote_identifier] = Post(**post_data) - return results.values() + self.instances = results.values() -class RedditScheduler: - max_amount = RATE_LIMIT - max_user_amount = RATE_LIMIT / 4 - - def __init__(self, subreddits=[]): - if not subreddits: - self.subreddits = CollectionRule.objects.filter( - type=RuleTypeChoices.subreddit, - user__reddit_access_token__isnull=False, - user__reddit_refresh_token__isnull=False, - enabled=True, - ).order_by("last_suceeded")[:200] - else: - self.subreddits = subreddits - - def get_scheduled_rules(self): - rule_mapping = {} - current_amount = 0 - - for subreddit in self.subreddits: - user_pk = subreddit.user.pk - - if current_amount == self.max_amount: - break - - if user_pk in rule_mapping: - max_amount_reached = len(rule_mapping[user_pk]) == self.max_user_amount - - if max_amount_reached: - continue - - rule_mapping[user_pk].append(subreddit) - current_amount += 1 - - continue - - rule_mapping[user_pk] = [subreddit] - current_amount += 1 - - return list(rule_mapping.values()) - - -class RedditStream(Stream): +class RedditStream(PostStream): + rule_type = RuleTypeChoices.subreddit headers = {} user = None @@ -261,16 +223,13 @@ class RedditStream(Stream): return response.json() except JSONDecodeError as e: raise StreamParseException( - response=response, message=f"Failed parsing json" + response=response, message="Failed parsing json" ) from e -class RedditClient(Client): +class RedditClient(PostClient): stream = RedditStream - def __init__(self, rules=[]): - self.rules = rules - def __enter__(self): streams = [[self.stream(rule) for rule in batch] for batch in self.rules] rate_limitted = False @@ -324,13 +283,49 @@ class RedditClient(Client): finally: stream.rule.save() - def set_rule_error(self, rule, exception): - length = rule._meta.get_field("error").max_length - rule.error = exception.message[-length:] - rule.succeeded = False - - -class RedditCollector(Collector): +class RedditCollector(PostCollector): builder = RedditBuilder client = RedditClient + + +class RedditScheduler: + max_amount = RATE_LIMIT + max_user_amount = RATE_LIMIT / 4 + + def __init__(self, subreddits=[]): + if not subreddits: + self.subreddits = CollectionRule.objects.filter( + type=RuleTypeChoices.subreddit, + user__reddit_access_token__isnull=False, + user__reddit_refresh_token__isnull=False, + enabled=True, + ).order_by("last_suceeded")[:200] + else: + self.subreddits = subreddits + + def get_scheduled_rules(self): + rule_mapping = {} + current_amount = 0 + + for subreddit in self.subreddits: + user_pk = subreddit.user.pk + + if current_amount == self.max_amount: + break + + if user_pk in rule_mapping: + max_amount_reached = len(rule_mapping[user_pk]) == self.max_user_amount + + if max_amount_reached: + continue + + rule_mapping[user_pk].append(subreddit) + current_amount += 1 + + continue + + rule_mapping[user_pk] = [subreddit] + current_amount += 1 + + return list(rule_mapping.values()) diff --git a/src/newsreader/news/collection/tests/favicon/builder/tests.py b/src/newsreader/news/collection/tests/favicon/builder/tests.py index e8a1a34..d21f77e 100644 --- a/src/newsreader/news/collection/tests/favicon/builder/tests.py +++ b/src/newsreader/news/collection/tests/favicon/builder/tests.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock + from django.test import TestCase from newsreader.news.collection.favicon import FaviconBuilder @@ -12,8 +14,11 @@ class FaviconBuilderTestCase(TestCase): def test_simple(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, simple_mock)) as builder: + with FaviconBuilder(simple_mock, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.bbc.com/favicon.ico") @@ -22,24 +27,33 @@ class FaviconBuilderTestCase(TestCase): website_url="https://www.theguardian.com/", favicon=None ) - with FaviconBuilder((rule, mock_without_url)) as builder: + with FaviconBuilder(mock_without_url, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.theguardian.com/favicon.ico") def test_without_header(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_without_header)) as builder: + with FaviconBuilder(mock_without_header, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, None) def test_weird_path(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_with_weird_path)) as builder: + with FaviconBuilder(mock_with_weird_path, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals( rule.favicon, "https://www.theguardian.com/jabadaba/doe/favicon.ico" @@ -48,15 +62,21 @@ class FaviconBuilderTestCase(TestCase): def test_other_url(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_with_other_url)) as builder: + with FaviconBuilder(mock_with_other_url, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.theguardian.com/icon.png") def test_url_with_favicon_takes_precedence(self): rule = CollectionRuleFactory(favicon=None) - with FaviconBuilder((rule, mock_with_multiple_icons)) as builder: + with FaviconBuilder(mock_with_multiple_icons, Mock(rule=rule)) as builder: builder.build() + builder.save() + + rule.refresh_from_db() self.assertEquals(rule.favicon, "https://www.bbc.com/favicon.ico") diff --git a/src/newsreader/news/collection/tests/favicon/client/tests.py b/src/newsreader/news/collection/tests/favicon/client/tests.py index 717ee0c..85b8fa3 100644 --- a/src/newsreader/news/collection/tests/favicon/client/tests.py +++ b/src/newsreader/news/collection/tests/favicon/client/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase @@ -19,22 +19,22 @@ class FaviconClientTestCase(TestCase): def test_simple(self): rule = CollectionRuleFactory() - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.return_value = (simple_mock, stream) - with FaviconClient([(rule, stream)]) as client: - for rule, data in client: - self.assertEquals(rule.pk, rule.pk) - self.assertEquals(data, simple_mock) + with FaviconClient([stream]) as client: + for payload, stream in client: + self.assertEquals(stream.rule.pk, rule.pk) + self.assertEquals(payload, simple_mock) stream.read.assert_called_once_with() def test_client_catches_stream_exception(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass @@ -46,10 +46,10 @@ class FaviconClientTestCase(TestCase): def test_client_catches_stream_not_found_exception(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamNotFoundException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass @@ -61,10 +61,10 @@ class FaviconClientTestCase(TestCase): def test_client_catches_stream_denied_exception(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamDeniedException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass @@ -76,10 +76,10 @@ class FaviconClientTestCase(TestCase): def test_client_catches_stream_timed_out(self): rule = CollectionRuleFactory(error=None, succeeded=True) - stream = MagicMock(url="https://www.bbc.com") + stream = Mock(url="https://www.bbc.com", rule=rule) stream.read.side_effect = StreamTimeOutException - with FaviconClient([(rule, stream)]) as client: + with FaviconClient([stream]) as client: for rule, data in client: pass diff --git a/src/newsreader/news/collection/tests/favicon/collector/tests.py b/src/newsreader/news/collection/tests/favicon/collector/tests.py index 44254a5..cb73a7c 100644 --- a/src/newsreader/news/collection/tests/favicon/collector/tests.py +++ b/src/newsreader/news/collection/tests/favicon/collector/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase @@ -38,8 +38,8 @@ class FaviconCollectorTestCase(TestCase): def test_simple(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] - self.mocked_website_read.return_value = (website_mock, MagicMock()) + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] + self.mocked_website_read.return_value = (website_mock, Mock(rule=rule)) collector = FaviconCollector() collector.collect() @@ -54,8 +54,11 @@ class FaviconCollectorTestCase(TestCase): def test_empty_stream(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] - self.mocked_website_read.return_value = (BeautifulSoup("", "lxml"), MagicMock()) + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] + self.mocked_website_read.return_value = ( + BeautifulSoup("", "lxml"), + Mock(rule=rule), + ) collector = FaviconCollector() collector.collect() @@ -70,7 +73,7 @@ class FaviconCollectorTestCase(TestCase): def test_not_found(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamNotFoundException collector = FaviconCollector() @@ -86,7 +89,7 @@ class FaviconCollectorTestCase(TestCase): def test_denied(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamDeniedException collector = FaviconCollector() @@ -102,7 +105,7 @@ class FaviconCollectorTestCase(TestCase): def test_forbidden(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamForbiddenException collector = FaviconCollector() @@ -118,7 +121,7 @@ class FaviconCollectorTestCase(TestCase): def test_timed_out(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamTimeOutException collector = FaviconCollector() @@ -134,7 +137,7 @@ class FaviconCollectorTestCase(TestCase): def test_wrong_stream_content_type(self): rule = CollectionRuleFactory(succeeded=True, error=None) - self.mocked_feed_client.return_value = [(feed_mock, MagicMock(rule=rule))] + self.mocked_feed_client.return_value = [(feed_mock, Mock(rule=rule))] self.mocked_website_read.side_effect = StreamParseException collector = FaviconCollector() diff --git a/src/newsreader/news/collection/tests/feed/builder/tests.py b/src/newsreader/news/collection/tests/feed/builder/tests.py index 4a6eb69..571a7cd 100644 --- a/src/newsreader/news/collection/tests/feed/builder/tests.py +++ b/src/newsreader/news/collection/tests/feed/builder/tests.py @@ -1,5 +1,5 @@ from datetime import date, datetime, time -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase from django.utils import timezone @@ -24,9 +24,10 @@ class FeedBuilderTestCase(TestCase): def test_basic_entry(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -55,9 +56,10 @@ class FeedBuilderTestCase(TestCase): def test_multiple_entries(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((multiple_mock, mock_stream)) as builder: + with builder(multiple_mock, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -116,9 +118,10 @@ class FeedBuilderTestCase(TestCase): def test_entries_without_remote_identifier(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_identifier, mock_stream)) as builder: + with builder(mock_without_identifier, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -155,9 +158,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_publication_date(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_publish_date, mock_stream)) as builder: + with builder(mock_without_publish_date, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -187,9 +191,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_url(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_url, mock_stream)) as builder: + with builder(mock_without_url, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -213,9 +218,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_body(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_body, mock_stream)) as builder: + with builder(mock_without_body, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -247,9 +253,10 @@ class FeedBuilderTestCase(TestCase): def test_entry_without_author(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_author, mock_stream)) as builder: + with builder(mock_without_author, mock_stream) as builder: + builder.build() builder.save() posts = Post.objects.order_by("-publication_date") @@ -275,9 +282,10 @@ class FeedBuilderTestCase(TestCase): def test_empty_entries(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_without_entries, mock_stream)) as builder: + with builder(mock_without_entries, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -285,7 +293,7 @@ class FeedBuilderTestCase(TestCase): def test_update_entries(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) existing_first_post = FeedPostFactory.create( remote_identifier="28f79ae4-8f9a-11e9-b143-00163ef6bee7", rule=rule @@ -295,7 +303,8 @@ class FeedBuilderTestCase(TestCase): remote_identifier="a5479c66-8fae-11e9-8422-00163ef6bee7", rule=rule ) - with builder((mock_with_update_entries, mock_stream)) as builder: + with builder(mock_with_update_entries, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 3) @@ -315,9 +324,10 @@ class FeedBuilderTestCase(TestCase): def test_html_sanitizing(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_html, mock_stream)) as builder: + with builder(mock_with_html, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -337,9 +347,10 @@ class FeedBuilderTestCase(TestCase): def test_long_author_text_is_truncated(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_long_author, mock_stream)) as builder: + with builder(mock_with_long_author, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -351,9 +362,10 @@ class FeedBuilderTestCase(TestCase): def test_long_title_text_is_truncated(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_long_title, mock_stream)) as builder: + with builder(mock_with_long_title, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -366,9 +378,10 @@ class FeedBuilderTestCase(TestCase): def test_long_title_exotic_title(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_long_exotic_title, mock_stream)) as builder: + with builder(mock_with_long_exotic_title, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -381,9 +394,10 @@ class FeedBuilderTestCase(TestCase): def test_content_detail_is_prioritized_if_longer(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_longer_content_detail, mock_stream)) as builder: + with builder(mock_with_longer_content_detail, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -398,9 +412,10 @@ class FeedBuilderTestCase(TestCase): def test_content_detail_is_not_prioritized_if_shorter(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_shorter_content_detail, mock_stream)) as builder: + with builder(mock_with_shorter_content_detail, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -414,9 +429,10 @@ class FeedBuilderTestCase(TestCase): def test_content_detail_is_concatinated(self): builder = FeedBuilder rule = FeedFactory() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) - with builder((mock_with_multiple_content_detail, mock_stream)) as builder: + with builder(mock_with_multiple_content_detail, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() diff --git a/src/newsreader/news/collection/tests/feed/client/tests.py b/src/newsreader/news/collection/tests/feed/client/tests.py index 24eb214..9a2365e 100644 --- a/src/newsreader/news/collection/tests/feed/client/tests.py +++ b/src/newsreader/news/collection/tests/feed/client/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase from django.utils.lorem_ipsum import words @@ -28,7 +28,7 @@ class FeedClientTestCase(TestCase): def test_client_retrieves_single_rules(self): rule = FeedFactory.create() - mock_stream = MagicMock(rule=rule) + mock_stream = Mock(rule=rule) self.mocked_read.return_value = (simple_mock, mock_stream) diff --git a/src/newsreader/news/collection/tests/feed/collector/tests.py b/src/newsreader/news/collection/tests/feed/collector/tests.py index 5a1bac1..048d618 100644 --- a/src/newsreader/news/collection/tests/feed/collector/tests.py +++ b/src/newsreader/news/collection/tests/feed/collector/tests.py @@ -1,6 +1,6 @@ from datetime import date, datetime, time from time import struct_time -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase from django.utils import timezone @@ -56,7 +56,7 @@ class FeedCollectorTestCase(TestCase): @freeze_time("2019-10-30 12:30:00") def test_emtpy_batch(self): - self.mocked_fetch.return_value = MagicMock() + self.mocked_fetch.return_value = Mock() self.mocked_parse.return_value = empty_mock rule = FeedFactory() diff --git a/src/newsreader/news/collection/tests/feed/stream/tests.py b/src/newsreader/news/collection/tests/feed/stream/tests.py index 82a09a3..f827c15 100644 --- a/src/newsreader/news/collection/tests/feed/stream/tests.py +++ b/src/newsreader/news/collection/tests/feed/stream/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase @@ -27,7 +27,7 @@ class FeedStreamTestCase(TestCase): patch.stopall() def test_simple_stream(self): - self.mocked_fetch.return_value = MagicMock(content=simple_mock) + self.mocked_fetch.return_value = Mock(content=simple_mock) rule = FeedFactory() stream = FeedStream(rule) @@ -95,7 +95,7 @@ class FeedStreamTestCase(TestCase): @patch("newsreader.news.collection.feed.parse") def test_stream_raises_parse_exception(self, mocked_parse): - self.mocked_fetch.return_value = MagicMock() + self.mocked_fetch.return_value = Mock() mocked_parse.side_effect = TypeError rule = FeedFactory() diff --git a/src/newsreader/news/collection/tests/reddit/builder/tests.py b/src/newsreader/news/collection/tests/reddit/builder/tests.py index 9c1a046..11cf549 100644 --- a/src/newsreader/news/collection/tests/reddit/builder/tests.py +++ b/src/newsreader/news/collection/tests/reddit/builder/tests.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase @@ -20,9 +20,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -65,9 +66,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((empty_mock, mock_stream)) as builder: + with builder(empty_mock, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -76,9 +78,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((unknown_mock, mock_stream)) as builder: + with builder(unknown_mock, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -95,9 +98,10 @@ class RedditBuilderTestCase(TestCase): ) builder = RedditBuilder - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -132,9 +136,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((unsanitized_mock, mock_stream)) as builder: + with builder(unsanitized_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -149,9 +154,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((author_mock, mock_stream)) as builder: + with builder(author_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -166,9 +172,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((title_mock, mock_stream)) as builder: + with builder(title_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -186,9 +193,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((duplicate_mock, mock_stream)) as builder: + with builder(duplicate_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -200,13 +208,14 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) duplicate_post = RedditPostFactory( remote_identifier="hm0qct", rule=subreddit, title="foo" ) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -231,9 +240,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((image_mock, mock_stream)) as builder: + with builder(image_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -262,9 +272,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((external_image_mock, mock_stream)) as builder: + with builder(external_image_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -302,9 +313,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((video_mock, mock_stream)) as builder: + with builder(video_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -328,9 +340,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((external_video_mock, mock_stream)) as builder: + with builder(external_video_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -354,9 +367,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((external_gifv_mock, mock_stream)) as builder: + with builder(external_gifv_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get() @@ -376,9 +390,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() post = Post.objects.get(remote_identifier="hngsj8") @@ -400,9 +415,10 @@ class RedditBuilderTestCase(TestCase): builder = RedditBuilder subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) - with builder((unknown_mock, mock_stream)) as builder: + with builder(unknown_mock, mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) diff --git a/src/newsreader/news/collection/tests/tests.py b/src/newsreader/news/collection/tests/tests.py index 363e0b5..c7f0bb0 100644 --- a/src/newsreader/news/collection/tests/tests.py +++ b/src/newsreader/news/collection/tests/tests.py @@ -1,10 +1,9 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase from bs4 import BeautifulSoup -from newsreader.news.collection.base import URLBuilder, WebsiteStream from newsreader.news.collection.exceptions import ( StreamDeniedException, StreamException, @@ -13,6 +12,7 @@ from newsreader.news.collection.exceptions import ( StreamParseException, StreamTimeOutException, ) +from newsreader.news.collection.favicon import WebsiteStream, WebsiteURLBuilder from newsreader.news.collection.tests.factories import CollectionRuleFactory from .mocks import feed_mock_without_link, simple_feed_mock, simple_mock @@ -20,117 +20,125 @@ from .mocks import feed_mock_without_link, simple_feed_mock, simple_mock class WebsiteStreamTestCase(TestCase): def setUp(self): - self.patched_fetch = patch("newsreader.news.collection.base.fetch") + self.patched_fetch = patch("newsreader.news.collection.favicon.fetch") self.mocked_fetch = self.patched_fetch.start() def tearDown(self): patch.stopall() def test_simple(self): - self.mocked_fetch.return_value = MagicMock(content=simple_mock) + self.mocked_fetch.return_value = Mock(content=simple_mock) - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) return_value = stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) - self.assertEquals(return_value, (BeautifulSoup(simple_mock, "lxml"), stream)) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") + self.assertEquals( + return_value, (BeautifulSoup(simple_mock, features="lxml"), stream) + ) def test_raises_exception(self): self.mocked_fetch.side_effect = StreamException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_raises_denied_exception(self): self.mocked_fetch.side_effect = StreamDeniedException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamDeniedException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_raises_stream_not_found_exception(self): self.mocked_fetch.side_effect = StreamNotFoundException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamNotFoundException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_stream_raises_time_out_exception(self): self.mocked_fetch.side_effect = StreamTimeOutException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamTimeOutException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") def test_stream_raises_forbidden_exception(self): self.mocked_fetch.side_effect = StreamForbiddenException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamForbiddenException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") - @patch("newsreader.news.collection.base.WebsiteStream.parse") + @patch("newsreader.news.collection.favicon.WebsiteStream.parse") def test_stream_raises_parse_exception(self, mocked_parse): - self.mocked_fetch.return_value = MagicMock() + self.mocked_fetch.return_value = Mock() mocked_parse.side_effect = StreamParseException - rule = CollectionRuleFactory() - stream = WebsiteStream(rule.url) + rule = CollectionRuleFactory(website_url="https://www.bbc.co.uk/news/") + stream = WebsiteStream(rule) with self.assertRaises(StreamParseException): stream.read() - self.mocked_fetch.assert_called_once_with(rule.url) + self.mocked_fetch.assert_called_once_with("https://www.bbc.co.uk/news/") -class URLBuilderTestCase(TestCase): +class WebsiteURLBuilderTestCase(TestCase): def test_simple(self): initial_rule = CollectionRuleFactory() - with URLBuilder((simple_feed_mock, MagicMock(rule=initial_rule))) as builder: - rule, url = builder.build() + with WebsiteURLBuilder(simple_feed_mock, Mock(rule=initial_rule)) as builder: + builder.build() + builder.save() - self.assertEquals(rule.pk, initial_rule.pk) - self.assertEquals(url, "https://www.bbc.co.uk/news/") + initial_rule.refresh_from_db() + + self.assertEquals(initial_rule.website_url, "https://www.bbc.co.uk/news/") def test_no_link(self): - initial_rule = CollectionRuleFactory() + initial_rule = CollectionRuleFactory(website_url=None) - with URLBuilder( - (feed_mock_without_link, MagicMock(rule=initial_rule)) + with WebsiteURLBuilder( + feed_mock_without_link, Mock(rule=initial_rule) ) as builder: - rule, url = builder.build() + builder.build() + builder.save() - self.assertEquals(rule.pk, initial_rule.pk) - self.assertEquals(url, None) + initial_rule.refresh_from_db() + + self.assertEquals(initial_rule.website_url, None) def test_no_data(self): - initial_rule = CollectionRuleFactory() + initial_rule = CollectionRuleFactory(website_url=None) - with URLBuilder((None, MagicMock(rule=initial_rule))) as builder: - rule, url = builder.build() + with WebsiteURLBuilder(None, Mock(rule=initial_rule)) as builder: + builder.build() + builder.save() - self.assertEquals(rule.pk, initial_rule.pk) - self.assertEquals(url, None) + initial_rule.refresh_from_db() + + self.assertEquals(initial_rule.website_url, None) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 19fdce3..8e08869 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -34,7 +34,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((simple_mock, mock_stream)) as builder: + with builder(simple_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -83,7 +84,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((image_mock, mock_stream)) as builder: + with builder(image_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -123,7 +125,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((video_mock, mock_stream)) as builder: + with builder(video_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -165,7 +168,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((video_without_bitrate_mock, mock_stream)) as builder: + with builder(video_without_bitrate_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -186,7 +190,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((gif_mock, mock_stream)) as builder: + with builder(gif_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -211,7 +216,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((retweet_mock, mock_stream)) as builder: + with builder(retweet_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -246,7 +252,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((quoted_mock, mock_stream)) as builder: + with builder(quoted_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} @@ -276,7 +283,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder(([], mock_stream)) as builder: + with builder([], mock_stream) as builder: + builder.build() builder.save() self.assertEquals(Post.objects.count(), 0) @@ -287,7 +295,8 @@ class TwitterBuilderTestCase(TestCase): profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") mock_stream = MagicMock(rule=profile) - with builder((unsanitized_mock, mock_stream)) as builder: + with builder(unsanitized_mock, mock_stream) as builder: + builder.build() builder.save() posts = {post.remote_identifier: post for post in Post.objects.all()} diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index b0f08cc..3d1c54b 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -8,7 +8,12 @@ import pytz from ftfy import fix_text -from newsreader.news.collection.base import Builder, Client, Collector, Stream +from newsreader.news.collection.base import ( + PostBuilder, + PostClient, + PostCollector, + PostStream, +) from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices from newsreader.news.collection.utils import truncate_text from newsreader.news.core.models import Post @@ -20,25 +25,14 @@ TWITTER_URL = "https://twitter.com" TWITTER_API_URL = "https://api.twitter.com/1.1" -class TwitterScheduler: - pass +class TwitterBuilder(PostBuilder): + rule_type = RuleTypeChoices.twitter - -class TwitterBuilder(Builder): - rule__type = RuleTypeChoices.twitter - - def create_posts(self, stream): - data, stream = stream - - if not data: - return - - self.instances = self.build(data, stream.rule) - - def build(self, posts, rule): + def build(self): results = {} + rule = self.stream.rule - for post in posts: + for post in self.payload: remote_identifier = post["id_str"] url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" @@ -83,7 +77,7 @@ class TwitterBuilder(Builder): results[remote_identifier] = Post(**data) - return results.values() + self.instances = results.values() def get_media_entities(self, post): media_entities = post["extended_entities"]["media"] @@ -133,13 +127,17 @@ class TwitterBuilder(Builder): return formatted_entities -class TwitterStream(Stream): +class TwitterStream(PostStream): pass -class TwitterClient(Client): +class TwitterClient(PostClient): pass -class TwitterCollector(Collector): +class TwitterCollector(PostCollector): + pass + + +class TwitterScheduler: pass From 396d36f8667bb20fe3322b495271f1dd5d78262c Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 13 Sep 2020 13:35:08 +0200 Subject: [PATCH 16/69] Use simple Mock instead of MagicMock --- .../collection/tests/reddit/client/tests.py | 6 +++--- .../collection/tests/twitter/builder/tests.py | 20 +++++++++---------- .../news/collection/tests/utils/tests.py | 14 ++++++------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/newsreader/news/collection/tests/reddit/client/tests.py b/src/newsreader/news/collection/tests/reddit/client/tests.py index f2ee84d..4dcc10f 100644 --- a/src/newsreader/news/collection/tests/reddit/client/tests.py +++ b/src/newsreader/news/collection/tests/reddit/client/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from uuid import uuid4 from django.test import TestCase @@ -31,7 +31,7 @@ class RedditClientTestCase(TestCase): def test_client_retrieves_single_rules(self): subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) self.mocked_read.return_value = (simple_mock, mock_stream) @@ -150,7 +150,7 @@ class RedditClientTestCase(TestCase): def test_client_catches_long_exception_text(self): subreddit = SubredditFactory() - mock_stream = MagicMock(rule=subreddit) + mock_stream = Mock(rule=subreddit) self.mocked_read.side_effect = StreamParseException(message=words(1000)) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 8e08869..eba3e65 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import MagicMock +from unittest.mock import Mock from django.test import TestCase from django.utils.html import format_html @@ -32,7 +32,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(simple_mock, mock_stream) as builder: builder.build() @@ -82,7 +82,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(image_mock, mock_stream) as builder: builder.build() @@ -123,7 +123,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(video_mock, mock_stream) as builder: builder.build() @@ -166,7 +166,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(video_without_bitrate_mock, mock_stream) as builder: builder.build() @@ -188,7 +188,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(gif_mock, mock_stream) as builder: builder.build() @@ -214,7 +214,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(retweet_mock, mock_stream) as builder: builder.build() @@ -250,7 +250,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(quoted_mock, mock_stream) as builder: builder.build() @@ -281,7 +281,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder([], mock_stream) as builder: builder.build() @@ -293,7 +293,7 @@ class TwitterBuilderTestCase(TestCase): builder = TwitterBuilder profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") - mock_stream = MagicMock(rule=profile) + mock_stream = Mock(rule=profile) with builder(unsanitized_mock, mock_stream) as builder: builder.build() diff --git a/src/newsreader/news/collection/tests/utils/tests.py b/src/newsreader/news/collection/tests/utils/tests.py index 10013c3..e88d1bf 100644 --- a/src/newsreader/news/collection/tests/utils/tests.py +++ b/src/newsreader/news/collection/tests/utils/tests.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import Mock, patch from django.test import TestCase @@ -19,7 +19,7 @@ from newsreader.news.collection.utils import fetch, post class HelperFunctionTestCase: def test_simple(self): - self.mocked_method.return_value = MagicMock(status_code=200, content="content") + self.mocked_method.return_value = Mock(status_code=200, content="content") url = "https://www.bbc.co.uk/news" response = self.method(url) @@ -27,7 +27,7 @@ class HelperFunctionTestCase: self.assertEquals(response.content, "content") def test_raises_not_found(self): - self.mocked_method.return_value = MagicMock(status_code=404) + self.mocked_method.return_value = Mock(status_code=404) url = "https://www.bbc.co.uk/news" @@ -35,7 +35,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_denied(self): - self.mocked_method.return_value = MagicMock(status_code=401) + self.mocked_method.return_value = Mock(status_code=401) url = "https://www.bbc.co.uk/news" @@ -43,7 +43,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_forbidden(self): - self.mocked_method.return_value = MagicMock(status_code=403) + self.mocked_method.return_value = Mock(status_code=403) url = "https://www.bbc.co.uk/news" @@ -51,7 +51,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_timed_out(self): - self.mocked_method.return_value = MagicMock(status_code=408) + self.mocked_method.return_value = Mock(status_code=408) url = "https://www.bbc.co.uk/news" @@ -99,7 +99,7 @@ class HelperFunctionTestCase: self.method(url) def test_raises_stream_error_on_too_many_requests(self): - self.mocked_method.return_value = MagicMock(status_code=429) + self.mocked_method.return_value = Mock(status_code=429) url = "https://www.bbc.co.uk/news" From 646c629327f9fec12d29d89469fd931f3133836a Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 13 Sep 2020 14:27:00 +0200 Subject: [PATCH 17/69] Fix memcached not linking to containers --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c7dc5ca..8ce24e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" volumes: postgres-data: static-files: @@ -16,7 +16,7 @@ services: rabbitmq: image: rabbitmq:3.7 memcached: - image: memcached:1.5.22 + image: memcached:1.6 ports: - "11211:11211" entrypoint: @@ -31,6 +31,7 @@ services: - DJANGO_SETTINGS_MODULE=newsreader.conf.docker depends_on: - rabbitmq + - memcached volumes: - .:/app django: @@ -41,9 +42,10 @@ services: environment: - DJANGO_SETTINGS_MODULE=newsreader.conf.docker ports: - - '8000:8000' + - "8000:8000" depends_on: - db + - memcached volumes: - .:/app - static-files:/app/src/newsreader/static From 5e53faf417e39e1b8b6c0d3108e0eb542c3a2beb Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 13 Sep 2020 15:00:31 +0200 Subject: [PATCH 18/69] Urlize tweet bodies --- .../collection/tests/twitter/builder/tests.py | 111 +++++++++++++++--- src/newsreader/news/collection/twitter.py | 26 ++-- 2 files changed, 110 insertions(+), 27 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index eba3e65..cc43c3c 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -2,7 +2,7 @@ from datetime import datetime from unittest.mock import Mock from django.test import TestCase -from django.utils.html import format_html +from django.utils.safestring import mark_safe import pytz @@ -46,11 +46,21 @@ class TwitterBuilderTestCase(TestCase): post = posts["1291528756373286914"] - full_text = "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" + full_text = ( + "@ArieNeoSC Here you go, goodnight!\n\n" + """https://t.co/trAcIxBMlX""" + ) self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncate_text(Post, "title", full_text)) - self.assertEquals(post.body, format_html(full_text)) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX", + ), + ) + self.assertEquals(post.body, mark_safe(full_text)) self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( @@ -66,7 +76,7 @@ class TwitterBuilderTestCase(TestCase): self.assertEquals(post.rule, profile) self.assertEquals(post.title, truncate_text(Post, "title", full_text)) - self.assertEquals(post.body, format_html(full_text)) + self.assertEquals(post.body, mark_safe(full_text)) self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( @@ -94,10 +104,8 @@ class TwitterBuilderTestCase(TestCase): post = posts["1269039237166321664"] - full_text = "_ https://t.co/VjEeDrL1iA" - self.assertEquals(post.rule, profile) - self.assertEquals(post.title, full_text) + self.assertEquals(post.title, "_ https://t.co/VjEeDrL1iA") self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( @@ -107,7 +115,11 @@ class TwitterBuilderTestCase(TestCase): post.publication_date, pytz.utc.localize(datetime(2020, 6, 5, 22, 51, 46)) ) - self.assertIn(full_text, post.body) + self.assertInHTML( + """https://t.co/VjEeDrL1iA""", + post.body, + count=1, + ) self.assertInHTML( """
1269039233072689152
""", post.body, @@ -141,11 +153,24 @@ class TwitterBuilderTestCase(TestCase): "Small enough to access hard-to-reach ore deposits, but with enough" " power to get through the tough jobs, Greycat\u2019s ROC perfectly" " complements any mining operation. \n\nDetails:" - " https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH" + """ https://t.co/2aH7qdOfSk""" + """ https://t.co/mZ8CAuq3SH""" ) self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncate_text(Post, "title", full_text)) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + fix_text( + "Small enough to access hard-to-reach ore deposits, but with enough" + " power to get through the tough jobs, Greycat\u2019s ROC perfectly" + " complements any mining operation. \n\nDetails:" + " https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH" + ), + ), + ) self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( @@ -208,7 +233,10 @@ class TwitterBuilderTestCase(TestCase): count=1, ) - self.assertIn("@Xenosystems https://t.co/wxvioLCJ6h", post.body) + self.assertIn( + """@Xenosystems https://t.co/wxvioLCJ6h""", + post.body, + ) def test_retweet_post(self): builder = TwitterBuilder @@ -241,7 +269,8 @@ class TwitterBuilderTestCase(TestCase): "Original tweet: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch" " 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease," " share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe!" - " \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\nhttps://t.co/j4QahHzbw4" + " \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\n" + """https://t.co/j4QahHzbw4""" ), post.body, ) @@ -265,14 +294,17 @@ class TwitterBuilderTestCase(TestCase): post = posts["1290801039075979264"] self.assertIn( - fix_text("Bonne nuit \ud83c\udf3a\ud83d\udeeb https://t.co/WyznJwCJLp"), + fix_text( + "Bonne nuit \ud83c\udf3a\ud83d\udeeb" + """ https://t.co/WyznJwCJLp""" + ), post.body, ) self.assertIn( fix_text( "Quoted tweet: #Starcitizen Le jeu est beau. Bonne nuit" - " @RobertsSpaceInd https://t.co/xCXun68V3r" + """ @RobertsSpaceInd https://t.co/xCXun68V3r""" ), post.body, ) @@ -306,16 +338,59 @@ class TwitterBuilderTestCase(TestCase): post = posts["1291528756373286914"] full_text = ( - "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" + "@ArieNeoSC Here you go, goodnight!\n\n" + """https://t.co/trAcIxBMlX""" "
" ) self.assertEquals(post.rule, profile) - self.assertEquals(post.title, truncate_text(Post, "title", full_text)) - self.assertEquals(post.body, format_html(full_text)) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX" + "
", + ), + ) + self.assertEquals(post.body, mark_safe(full_text)) self.assertInHTML("", post.body, count=0) self.assertInHTML("
", post.body, count=1) self.assertInHTML("", post.title, count=0) self.assertInHTML("
", post.title, count=1) + + def test_urlize_on_urls(self): + builder = TwitterBuilder + + profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + with builder(simple_mock, mock_stream) as builder: + builder.build() + builder.save() + + posts = {post.remote_identifier: post for post in Post.objects.all()} + + self.assertCountEqual( + ("1291528756373286914", "1288550304095416320"), posts.keys() + ) + + post = posts["1291528756373286914"] + + full_text = ( + "@ArieNeoSC Here you go, goodnight!\n\n" + """https://t.co/trAcIxBMlX""" + ) + + self.assertEquals(post.rule, profile) + self.assertEquals( + post.title, + truncate_text( + Post, + "title", + "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX", + ), + ) + self.assertEquals(post.body, mark_safe(full_text)) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 3d1c54b..19fb812 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -2,7 +2,7 @@ import logging from datetime import datetime -from django.utils.html import format_html +from django.utils.html import format_html, urlize import pytz @@ -36,8 +36,10 @@ class TwitterBuilder(PostBuilder): remote_identifier = post["id_str"] url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" - body = post["full_text"] - title = truncate_text(Post, "title", self.sanitize_fragment(body)) + body = urlize(post["full_text"], nofollow=True) + title = truncate_text( + Post, "title", self.sanitize_fragment(post["full_text"]) + ) publication_date = pytz.utc.localize( datetime.strptime(post["created_at"], "%a %b %d %H:%M:%S +0000 %Y") @@ -52,15 +54,21 @@ class TwitterBuilder(PostBuilder): if "retweeted_status" in post: original_post = post["retweeted_status"] - body += format_html( - "Original tweet: {original_post}", - original_post=original_post["full_text"], + body += urlize( + format_html( + "Original tweet: {original_post}", + original_post=urlize(original_post["full_text"], nofollow=True), + ), + nofollow=True, ) if "quoted_status" in post: original_post = post["quoted_status"] - body += format_html( - "Quoted tweet: {original_post}", - original_post=original_post["full_text"], + body += urlize( + format_html( + "Quoted tweet: {original_post}", + original_post=original_post["full_text"], + ), + nofollow=True, ) body = self.sanitize_fragment(body) From d9999752bcec99445ad4a548288b90db8acc3458 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 13 Sep 2020 15:34:02 +0200 Subject: [PATCH 19/69] Fix reddit exception log --- src/newsreader/news/collection/reddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 82a4d96..b905f24 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -274,7 +274,7 @@ class RedditClient(PostClient): break except StreamException as e: logger.exception( - "Stream failed reading content from " f"{stream.rule.url}" + "Stream failed reading content from {stream.rule.url}" ) self.set_rule_error(stream.rule, e) From 7fe3af0115ae2bcc4eb511c78c6297aa7bea1d0a Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 13 Sep 2020 22:36:31 +0200 Subject: [PATCH 20/69] Add initial twitter scheduler & rename last_suceeded to last_run --- .../migrations/0011_auto_20200913_2101.py | 21 + src/newsreader/accounts/models.py | 3 + src/newsreader/fixtures/default-fixture.json | 8046 ++++++++--------- src/newsreader/fixtures/local/fixture.json | 12 +- src/newsreader/news/collection/admin.py | 9 +- src/newsreader/news/collection/base.py | 9 + src/newsreader/news/collection/choices.py | 2 +- src/newsreader/news/collection/feed.py | 2 +- .../migrations/0010_auto_20200913_2101.py | 24 + .../migrations/0011_auto_20200913_2157.py | 14 + src/newsreader/news/collection/models.py | 3 +- src/newsreader/news/collection/reddit.py | 8 +- .../news/collection/tests/factories.py | 4 +- .../collection/tests/feed/collector/tests.py | 48 +- .../tests/reddit/collector/tests.py | 4 +- .../collection/tests/reddit/test_scheduler.py | 16 +- .../collection/tests/twitter/builder/tests.py | 22 +- src/newsreader/news/collection/twitter.py | 60 +- 18 files changed, 4211 insertions(+), 4096 deletions(-) create mode 100644 src/newsreader/accounts/migrations/0011_auto_20200913_2101.py create mode 100644 src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py create mode 100644 src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py diff --git a/src/newsreader/accounts/migrations/0011_auto_20200913_2101.py b/src/newsreader/accounts/migrations/0011_auto_20200913_2101.py new file mode 100644 index 0000000..b6a83dd --- /dev/null +++ b/src/newsreader/accounts/migrations/0011_auto_20200913_2101.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.7 on 2020-09-13 19:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("accounts", "0010_auto_20200603_2230")] + + operations = [ + migrations.AddField( + model_name="user", + name="twitter_oauth_token", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="user", + name="twitter_oauth_token_secret", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/src/newsreader/accounts/models.py b/src/newsreader/accounts/models.py index b8aaa64..a54c375 100644 --- a/src/newsreader/accounts/models.py +++ b/src/newsreader/accounts/models.py @@ -53,6 +53,9 @@ class User(AbstractUser): reddit_refresh_token = models.CharField(max_length=255, blank=True, null=True) reddit_access_token = models.CharField(max_length=255, blank=True, null=True) + twitter_oauth_token = models.CharField(max_length=255, blank=True, null=True) + twitter_oauth_token_secret = models.CharField(max_length=255, blank=True, null=True) + username = None objects = UserManager() diff --git a/src/newsreader/fixtures/default-fixture.json b/src/newsreader/fixtures/default-fixture.json index 10d6416..1794742 100644 --- a/src/newsreader/fixtures/default-fixture.json +++ b/src/newsreader/fixtures/default-fixture.json @@ -1,4023 +1,4023 @@ -[ -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "admin", - "model": "logentry" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "auth", - "model": "permission" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "auth", - "model": "group" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "contenttypes", - "model": "contenttype" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "sessions", - "model": "session" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "crontabschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "intervalschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "periodictask" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "periodictasks" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "solarschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "django_celery_beat", - "model": "clockedschedule" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "registration", - "model": "registrationprofile" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "registration", - "model": "supervisedregistrationprofile" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "axes", - "model": "accessattempt" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "axes", - "model": "accesslog" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "accounts", - "model": "user" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "core", - "model": "post" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "core", - "model": "category" - } -}, -{ - "model": "contenttypes.contenttype", - "fields": { - "app_label": "collection", - "model": "collectionrule" - } -}, -{ - "model": "sessions.session", - "pk": "3sumq22krk8tsvexcs4b8czu82yhvuer", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-05-16T18:29:04.049Z" - } -}, -{ - "model": "sessions.session", - "pk": "8ix6bdwf2ywk0eir1hb062dhfh9xit85", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-07-21T19:36:54.530Z" - } -}, -{ - "model": "sessions.session", - "pk": "d4wophwpjm8z96doe8iddvhdv9yfafyx", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-06-07T19:45:49.727Z" - } -}, -{ - "model": "sessions.session", - "pk": "g23ziz66li5zx8nd8cewb3vevdxhjkm0", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-06-30T06:55:50.747Z" - } -}, -{ - "model": "sessions.session", - "pk": "jwn66dptmdkm6hom2ns3j288aaxqtyjd", - "fields": { - "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", - "expire_date": "2020-06-07T18:38:19.116Z" - } -}, -{ - "model": "sessions.session", - "pk": "wjz6kwg5e5ciemre0l0wwyrcwcj2gyg6", - "fields": { - "session_data": "MWU5ODBjY2QyOTFhMmRiY2QyYjQwZjQ3MmMwYmExYjBlYTkxNTcwODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiI0YWZkYTkxNzU5ZDBhZDZmMjg1ZTQyOGY0OTUxN2M5MTJhMmM5NWIyIn0=", - "expire_date": "2020-08-09T09:52:04.705Z" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 1, - "fields": { - "every": 5, - "period": "minutes" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 2, - "fields": { - "every": 15, - "period": "minutes" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 3, - "fields": { - "every": 30, - "period": "minutes" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 4, - "fields": { - "every": 1, - "period": "hours" - } -}, -{ - "model": "django_celery_beat.intervalschedule", - "pk": 5, - "fields": { - "every": 4, - "period": "hours" - } -}, -{ - "model": "django_celery_beat.crontabschedule", - "pk": 1, - "fields": { - "minute": "0", - "hour": "4", - "day_of_week": "*", - "day_of_month": "*", - "month_of_year": "*", - "timezone": "UTC" - } -}, -{ - "model": "django_celery_beat.periodictasks", - "pk": 1, - "fields": { - "last_update": "2020-07-26T09:47:48.298Z" - } -}, -{ - "model": "django_celery_beat.periodictask", - "pk": 1, - "fields": { - "name": "celery.backend_cleanup", - "task": "celery.backend_cleanup", - "interval": null, - "crontab": 1, - "solar": null, - "clocked": null, - "args": "[]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "headers": "{}", - "priority": null, - "expires": null, - "expire_seconds": 43200, - "one_off": false, - "start_time": null, - "enabled": true, - "last_run_at": "2020-07-26T09:47:48.322Z", - "total_run_count": 17, - "date_changed": "2020-07-26T09:47:50.362Z", - "description": "" - } -}, -{ - "model": "django_celery_beat.periodictask", - "pk": 10, - "fields": { - "name": "sonny@bakker.nl-collection-task", - "task": "FeedTask", - "interval": 5, - "crontab": null, - "solar": null, - "clocked": null, - "args": "[1]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "headers": "{}", - "priority": null, - "expires": null, - "expire_seconds": null, - "one_off": false, - "start_time": null, - "enabled": false, - "last_run_at": "2020-07-14T11:45:26.209Z", - "total_run_count": 307, - "date_changed": "2020-07-14T11:45:41.282Z", - "description": "" - } -}, -{ - "model": "django_celery_beat.periodictask", - "pk": 11, - "fields": { - "name": "Reddit collection task", - "task": "RedditTask", - "interval": 5, - "crontab": null, - "solar": null, - "clocked": null, - "args": "[]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "headers": "{}", - "priority": null, - "expires": null, - "expire_seconds": null, - "one_off": false, - "start_time": null, - "enabled": false, - "last_run_at": null, - "total_run_count": 4, - "date_changed": "2020-07-14T11:45:41.316Z", - "description": "" - } -}, -{ - "model": "core.post", - "pk": 3061, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:14:50.423Z", - "title": "Star Citizen: Question and Answer Thread", - "body": "

Welcome to the Star Citizen question and answer thread. Feel free to ask any questions you have related to SC here!

\n\n\n\n

Useful Links and Resources:

\n\n

Star Citizen Wiki - The biggest and best wiki resource dedicated to Star Citizen

\n\n

Star Citizen FAQ - Chances the answer you need is here.

\n\n

Discord Help Channel - Often times community members will be here to help you with issues.

\n\n

Referral Code Randomizer - Use this when creating a new account to get 5000 extra UEC.

\n\n

Download Star Citizen - Get the latest version of Star Citizen here

\n\n

Current Game Features - Click here to see what you can currently do in Star Citizen.

\n\n

Development Roadmap - The current development status of up and coming Star Citizen features.

\n\n

Pledge FAQ - Official FAQ regarding spending money on the game.

\n
", - "author": "UEE_Central_Computer", - "publication_date": "2020-07-20T14:00:10Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huk04t/star_citizen_question_and_answer_thread/", - "read": false, - "rule": 82, - "remote_identifier": "huk04t" - } -}, -{ - "model": "core.post", - "pk": 3062, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:33:37.019Z", - "title": "Peace and Quiet", - "body": "
\"Peace
", - "author": "SourMemeNZ", - "publication_date": "2020-07-20T14:09:49Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huk4ib/peace_and_quiet/", - "read": true, - "rule": 82, - "remote_identifier": "huk4ib" - } -}, -{ - "model": "core.post", - "pk": 3063, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:14:50.463Z", - "title": "Y'all are probably sick of em by now but here's my LEGO Mercury Star Runner (MSR).", - "body": "
\"Y'all
", - "author": "osamadabinman", - "publication_date": "2020-07-20T19:53:23Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hupzqa/yall_are_probably_sick_of_em_by_now_but_heres_my/", - "read": true, - "rule": 82, - "remote_identifier": "hupzqa" - } -}, -{ - "model": "core.post", - "pk": 3064, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:12.253Z", - "title": "Damned Space Invaders and their pixel weapons!", - "body": "
\"Damned
", - "author": "Akaradrin", - "publication_date": "2020-07-20T14:26:18Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hukckf/damned_space_invaders_and_their_pixel_weapons/", - "read": true, - "rule": 82, - "remote_identifier": "hukckf" - } -}, -{ - "model": "core.post", - "pk": 3065, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.578Z", - "title": "The sky is no longer the limit", - "body": "
\"The
", - "author": "CyberTill", - "publication_date": "2020-07-20T14:11:31Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huk5b8/the_sky_is_no_longer_the_limit/", - "read": false, - "rule": 82, - "remote_identifier": "huk5b8" - } -}, -{ - "model": "core.post", - "pk": 3066, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:23.282Z", - "title": "Terrapin Hover Mode Gameplay [Full Video in Comments]", - "body": "
", - "author": "Didactic_Tomato", - "publication_date": "2020-07-20T11:01:13Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hui1gv/terrapin_hover_mode_gameplay_full_video_in/", - "read": true, - "rule": 82, - "remote_identifier": "hui1gv" - } -}, -{ - "model": "core.post", - "pk": 3067, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:44.250Z", - "title": "honestly", - "body": "
\"honestly\"
", - "author": "Beatlead", - "publication_date": "2020-07-20T18:24:07Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huo96t/honestly/", - "read": true, - "rule": 82, - "remote_identifier": "huo96t" - } -}, -{ - "model": "core.post", - "pk": 3068, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.584Z", - "title": "As a paranoiac and tired of checking if door was closed, saved to f4 thoses \"security cam\" positions, could be usefull for larger ships :)", - "body": "", - "author": "icwiener__", - "publication_date": "2020-07-20T13:03:33Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hujchz/as_a_paranoiac_and_tired_of_checking_if_door_was/", - "read": false, - "rule": 82, - "remote_identifier": "hujchz" - } -}, -{ - "model": "core.post", - "pk": 3069, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:33:59.158Z", - "title": "Station Manager: \"You're too fat, we won't let you in, go and fall on Lorville. Thank you for your call!\" Me: \"okay :'(\"", - "body": "
\"Station
", - "author": "Shaman_N_One", - "publication_date": "2020-07-20T11:33:38Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huidlu/station_manager_youre_too_fat_we_wont_let_you_in/", - "read": true, - "rule": 82, - "remote_identifier": "huidlu" - } -}, -{ - "model": "core.post", - "pk": 3070, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.588Z", - "title": "[PTU Bug Hunt Request] Packet Loss", - "body": "", - "author": "Rainwalker007", - "publication_date": "2020-07-20T18:38:03Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huoicq/ptu_bug_hunt_request_packet_loss/", - "read": false, - "rule": 82, - "remote_identifier": "huoicq" - } -}, -{ - "model": "core.post", - "pk": 3071, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:17:52.092Z", - "title": "Anyone able to explain these \"trail frames\"?", - "body": "
\"Anyone
", - "author": "Abnormal_Sloth", - "publication_date": "2020-07-20T17:11:32Z", - "url": "https://www.reddit.com/r/starcitizen/comments/humyeq/anyone_able_to_explain_these_trail_frames/", - "read": true, - "rule": 82, - "remote_identifier": "humyeq" - } -}, -{ - "model": "core.post", - "pk": 3072, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.593Z", - "title": "#BringBackBugSmasher - A long forgotten legendary video content", - "body": "", - "author": "MasterBoring", - "publication_date": "2020-07-20T18:05:54Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hunx77/bringbackbugsmasher_a_long_forgotten_legendary/", - "read": false, - "rule": 82, - "remote_identifier": "hunx77" - } -}, -{ - "model": "core.post", - "pk": 3073, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:33:22.601Z", - "title": "Oracle Helmet [in-game screenshot; downsampled to 4k]", - "body": "
\"Oracle
", - "author": "mr-hasgaha", - "publication_date": "2020-07-20T17:39:34Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hung0b/oracle_helmet_ingame_screenshot_downsampled_to_4k/", - "read": true, - "rule": 82, - "remote_identifier": "hung0b" - } -}, -{ - "model": "core.post", - "pk": 3074, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:34:42.578Z", - "title": "Testing 3.10 - Gladius in decoupled mode", - "body": "
", - "author": "DarkConstant", - "publication_date": "2020-07-19T21:26:52Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hu6f1h/testing_310_gladius_in_decoupled_mode/", - "read": true, - "rule": 82, - "remote_identifier": "hu6f1h" - } -}, -{ - "model": "core.post", - "pk": 3075, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:34:29.424Z", - "title": "Day 3, I can't stop taking pictures with my Carrack. Send help", - "body": "
\"Day
", - "author": "CyberTill", - "publication_date": "2020-07-20T01:58:15Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huazyy/day_3_i_cant_stop_taking_pictures_with_my_carrack/", - "read": true, - "rule": 82, - "remote_identifier": "huazyy" - } -}, -{ - "model": "core.post", - "pk": 3076, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.602Z", - "title": "I used to enjoy flying between the buildings of new babbage, I mean before the NFZ \"improvement\"", - "body": "
\"I
", - "author": "shoeii", - "publication_date": "2020-07-20T16:40:26Z", - "url": "https://www.reddit.com/r/starcitizen/comments/humet2/i_used_to_enjoy_flying_between_the_buildings_of/", - "read": false, - "rule": 82, - "remote_identifier": "humet2" - } -}, -{ - "model": "core.post", - "pk": 3077, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-21T20:18:04.237Z", - "title": "Thank you CIG for updated heightmaps and render distances", - "body": "
\"Thank
", - "author": "u7f76", - "publication_date": "2020-07-19T23:38:22Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hu8pwf/thank_you_cig_for_updated_heightmaps_and_render/", - "read": true, - "rule": 82, - "remote_identifier": "hu8pwf" - } -}, -{ - "model": "core.post", - "pk": 3078, - "fields": { - "created": "2020-07-20T19:32:35.562Z", - "modified": "2020-07-20T19:32:35.607Z", - "title": "This Week in Star Citizen | July 20th 2020", - "body": "", - "author": "ivtiprogamer", - "publication_date": "2020-07-20T19:50:29Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hupxnt/this_week_in_star_citizen_july_20th_2020/", - "read": false, - "rule": 82, - "remote_identifier": "hupxnt" - } -}, -{ - "model": "core.post", - "pk": 3079, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:34:36.068Z", - "title": "Bravo CIG lighting team! Noticeable improvements to all around environment lighting in 3.10", - "body": "
\"Bravo
", - "author": "u7f76", - "publication_date": "2020-07-20T00:02:23Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hu94o0/bravo_cig_lighting_team_noticeable_improvements/", - "read": true, - "rule": 82, - "remote_identifier": "hu94o0" - } -}, -{ - "model": "core.post", - "pk": 3080, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.613Z", - "title": "Thick", - "body": "
\"Thick\"
", - "author": "burgerbagel", - "publication_date": "2020-07-20T16:24:38Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hum50f/thick/", - "read": false, - "rule": 82, - "remote_identifier": "hum50f" - } -}, -{ - "model": "core.post", - "pk": 3081, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:34:19.763Z", - "title": "Soon\u2122", - "body": "
\"Soon\u2122\"
", - "author": "Mistralette", - "publication_date": "2020-07-20T05:54:09Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hueg01/soon/", - "read": true, - "rule": 82, - "remote_identifier": "hueg01" - } -}, -{ - "model": "core.post", - "pk": 3082, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.618Z", - "title": "On the prowl", - "body": "
\"On
", - "author": "SaraCaterina", - "publication_date": "2020-07-20T16:37:03Z", - "url": "https://www.reddit.com/r/starcitizen/comments/humcmb/on_the_prowl/", - "read": false, - "rule": 82, - "remote_identifier": "humcmb" - } -}, -{ - "model": "core.post", - "pk": 3083, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:34:07.272Z", - "title": "The Hills Have Eyes", - "body": "
\"The
", - "author": "FallenLordik", - "publication_date": "2020-07-20T11:19:19Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hui8ao/the_hills_have_eyes/", - "read": true, - "rule": 82, - "remote_identifier": "hui8ao" - } -}, -{ - "model": "core.post", - "pk": 3084, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.623Z", - "title": "Worried about longer loading screens? Hit ~ and do r_displayinfo 3", - "body": "
\"Worried
", - "author": "kristokn", - "publication_date": "2020-07-20T10:09:53Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huhif1/worried_about_longer_loading_screens_hit_and_do_r/", - "read": false, - "rule": 82, - "remote_identifier": "huhif1" - } -}, -{ - "model": "core.post", - "pk": 3085, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.625Z", - "title": "My contribution to the wallpaper contest... click for the full effect (3440x1440)", - "body": "
\"My
", - "author": "Dougie_Juice", - "publication_date": "2020-07-20T20:02:31Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huq655/my_contribution_to_the_wallpaper_contest_click/", - "read": false, - "rule": 82, - "remote_identifier": "huq655" - } -}, -{ - "model": "core.post", - "pk": 3086, - "fields": { - "created": "2020-07-20T19:32:35.563Z", - "modified": "2020-07-20T19:32:35.627Z", - "title": "Star Citizen: The Onion (Parody Project)", - "body": "", - "author": "BroadOne", - "publication_date": "2020-07-20T19:19:20Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hupbkj/star_citizen_the_onion_parody_project/", - "read": false, - "rule": 82, - "remote_identifier": "hupbkj" - } -}, -{ - "model": "core.post", - "pk": 3087, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.637Z", - "title": "perfect day to sunbathe", - "body": "
", - "author": "Pedrica1", - "publication_date": "2020-07-20T18:08:17Z", - "url": "https://www.reddit.com/r/aww/comments/hunysb/perfect_day_to_sunbathe/", - "read": false, - "rule": 81, - "remote_identifier": "hunysb" - } -}, -{ - "model": "core.post", - "pk": 3088, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.639Z", - "title": "My dogs face when he sees I'm home", - "body": "
", - "author": "NewReddit_WhoDis", - "publication_date": "2020-07-20T16:45:21Z", - "url": "https://www.reddit.com/r/aww/comments/humhxa/my_dogs_face_when_he_sees_im_home/", - "read": false, - "rule": 81, - "remote_identifier": "humhxa" - } -}, -{ - "model": "core.post", - "pk": 3089, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.641Z", - "title": "Cow loves the scritch machine", - "body": "
", - "author": "Der_Ist", - "publication_date": "2020-07-20T17:36:16Z", - "url": "https://www.reddit.com/r/aww/comments/hundvo/cow_loves_the_scritch_machine/", - "read": false, - "rule": 81, - "remote_identifier": "hundvo" - } -}, -{ - "model": "core.post", - "pk": 3090, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.643Z", - "title": "Can I sit next to you ?", - "body": "
", - "author": "wheezy098", - "publication_date": "2020-07-20T17:55:10Z", - "url": "https://www.reddit.com/r/aww/comments/hunq5h/can_i_sit_next_to_you/", - "read": false, - "rule": 81, - "remote_identifier": "hunq5h" - } -}, -{ - "model": "core.post", - "pk": 3091, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.645Z", - "title": "IS THAT A CUSTOMER? flop flop flop flop .... \" Can I uhh... help you sir?\"", - "body": "
", - "author": "MBMV", - "publication_date": "2020-07-20T12:50:40Z", - "url": "https://www.reddit.com/r/aww/comments/huj7g3/is_that_a_customer_flop_flop_flop_flop_can_i_uhh/", - "read": false, - "rule": 81, - "remote_identifier": "huj7g3" - } -}, -{ - "model": "core.post", - "pk": 3092, - "fields": { - "created": "2020-07-20T19:32:35.635Z", - "modified": "2020-07-20T19:32:35.647Z", - "title": "Good Boy turned Disney Princess", - "body": "
", - "author": "Sauwercraud", - "publication_date": "2020-07-20T18:40:05Z", - "url": "https://www.reddit.com/r/aww/comments/huojq0/good_boy_turned_disney_princess/", - "read": false, - "rule": 81, - "remote_identifier": "huojq0" - } -}, -{ - "model": "core.post", - "pk": 3093, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.649Z", - "title": "Kitty loop", - "body": "
", - "author": "Dlatrex", - "publication_date": "2020-07-20T12:54:02Z", - "url": "https://www.reddit.com/r/aww/comments/huj8s6/kitty_loop/", - "read": false, - "rule": 81, - "remote_identifier": "huj8s6" - } -}, -{ - "model": "core.post", - "pk": 3094, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.652Z", - "title": "if i fits i sits", - "body": "
", - "author": "jasontaken", - "publication_date": "2020-07-20T16:38:32Z", - "url": "https://www.reddit.com/r/aww/comments/humdlf/if_i_fits_i_sits/", - "read": false, - "rule": 81, - "remote_identifier": "humdlf" - } -}, -{ - "model": "core.post", - "pk": 3095, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.654Z", - "title": "Isn\u2019t she Adorable !", - "body": "
\"Isn\u2019t
", - "author": "MunchyMac", - "publication_date": "2020-07-20T16:18:05Z", - "url": "https://www.reddit.com/r/aww/comments/hum133/isnt_she_adorable/", - "read": false, - "rule": 81, - "remote_identifier": "hum133" - } -}, -{ - "model": "core.post", - "pk": 3096, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.655Z", - "title": "Thank you mama (\u2283\uff61\u2022\u0301\u203f\u2022\u0300\uff61)\u2283", - "body": "
", - "author": "AnoushkaSingh", - "publication_date": "2020-07-20T13:35:51Z", - "url": "https://www.reddit.com/r/aww/comments/hujpxy/thank_you_mama/", - "read": false, - "rule": 81, - "remote_identifier": "hujpxy" - } -}, -{ - "model": "core.post", - "pk": 3097, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.657Z", - "title": "I WANT TO HUG HIM SO BAD!!!", - "body": "
", - "author": "BATMAN_5777", - "publication_date": "2020-07-20T18:25:20Z", - "url": "https://www.reddit.com/r/aww/comments/huo9z4/i_want_to_hug_him_so_bad/", - "read": false, - "rule": 81, - "remote_identifier": "huo9z4" - } -}, -{ - "model": "core.post", - "pk": 3098, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.659Z", - "title": "Before and after being called a good boy", - "body": "
\"Before
", - "author": "vladgrinch", - "publication_date": "2020-07-20T10:48:40Z", - "url": "https://www.reddit.com/r/aww/comments/huhwu9/before_and_after_being_called_a_good_boy/", - "read": false, - "rule": 81, - "remote_identifier": "huhwu9" - } -}, -{ - "model": "core.post", - "pk": 3099, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.662Z", - "title": "My fianc\u00e9 has wanted a dog his whole life. This is his college graduation present. Welcome home Maple!", - "body": "
\"My
", - "author": "AlexisaurusRex", - "publication_date": "2020-07-20T17:57:25Z", - "url": "https://www.reddit.com/r/aww/comments/hunrie/my_fianc\u00e9_has_wanted_a_dog_his_whole_life_this_is/", - "read": false, - "rule": 81, - "remote_identifier": "hunrie" - } -}, -{ - "model": "core.post", - "pk": 3100, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.664Z", - "title": "Cute burro.", - "body": "
\"Cute
", - "author": "Craftmine101", - "publication_date": "2020-07-20T13:45:32Z", - "url": "https://www.reddit.com/r/aww/comments/huju40/cute_burro/", - "read": false, - "rule": 81, - "remote_identifier": "huju40" - } -}, -{ - "model": "core.post", - "pk": 3101, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.666Z", - "title": "I've never seen anyone dance better than that turtle.", - "body": "
", - "author": "Ashley1023", - "publication_date": "2020-07-20T18:07:30Z", - "url": "https://www.reddit.com/r/aww/comments/hunya8/ive_never_seen_anyone_dance_better_than_that/", - "read": false, - "rule": 81, - "remote_identifier": "hunya8" - } -}, -{ - "model": "core.post", - "pk": 3102, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.669Z", - "title": "Someone\u2019s going to be quite surprised when he realizes all this new stuff isn\u2019t for him!", - "body": "
\"Someone\u2019s
", - "author": "molly590", - "publication_date": "2020-07-20T15:46:21Z", - "url": "https://www.reddit.com/r/aww/comments/hulikg/someones_going_to_be_quite_surprised_when_he/", - "read": false, - "rule": 81, - "remote_identifier": "hulikg" - } -}, -{ - "model": "core.post", - "pk": 3103, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.671Z", - "title": "my aunt asked me to paint her puppy and I think it turned out so cute!!!", - "body": "
\"my
", - "author": "PineappleLightt", - "publication_date": "2020-07-20T16:39:37Z", - "url": "https://www.reddit.com/r/aww/comments/humea0/my_aunt_asked_me_to_paint_her_puppy_and_i_think/", - "read": false, - "rule": 81, - "remote_identifier": "humea0" - } -}, -{ - "model": "core.post", - "pk": 3104, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.673Z", - "title": "Master Assassin", - "body": "
\"Master
", - "author": "LauWalker", - "publication_date": "2020-07-20T18:47:52Z", - "url": "https://www.reddit.com/r/aww/comments/huop8a/master_assassin/", - "read": false, - "rule": 81, - "remote_identifier": "huop8a" - } -}, -{ - "model": "core.post", - "pk": 3105, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.675Z", - "title": "Every time this tank cleaner cleans out the aquarium, this fish swims over to him looking for pets", - "body": "", - "author": "unnaturalorder", - "publication_date": "2020-07-20T05:29:30Z", - "url": "https://www.reddit.com/r/aww/comments/hue3r0/every_time_this_tank_cleaner_cleans_out_the/", - "read": false, - "rule": 81, - "remote_identifier": "hue3r0" - } -}, -{ - "model": "core.post", - "pk": 3106, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.678Z", - "title": "My girlfriend sent me this while I was at work. And here I was thinking the perfect picture of our dog didn't exist", - "body": "", - "author": "Khuma-zi_Eldrama", - "publication_date": "2020-07-20T19:22:48Z", - "url": "https://www.reddit.com/r/aww/comments/hupdz8/my_girlfriend_sent_me_this_while_i_was_at_work/", - "read": false, - "rule": 81, - "remote_identifier": "hupdz8" - } -}, -{ - "model": "core.post", - "pk": 3107, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.680Z", - "title": "My first ever post, everyone meet my new baby girl Kiora! I\u2019m so in love with her\ud83e\udd7a\ud83d\udcab", - "body": "
\"My
", - "author": "Dumpling2463", - "publication_date": "2020-07-20T05:34:29Z", - "url": "https://www.reddit.com/r/aww/comments/hue6dx/my_first_ever_post_everyone_meet_my_new_baby_girl/", - "read": false, - "rule": 81, - "remote_identifier": "hue6dx" - } -}, -{ - "model": "core.post", - "pk": 3108, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.682Z", - "title": "Dog splashing in water", - "body": "", - "author": "TheRikari", - "publication_date": "2020-07-20T15:44:02Z", - "url": "https://www.reddit.com/r/aww/comments/hulh8k/dog_splashing_in_water/", - "read": false, - "rule": 81, - "remote_identifier": "hulh8k" - } -}, -{ - "model": "core.post", - "pk": 3109, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.685Z", - "title": "They say taking breaks is the key to productivity!", - "body": "
", - "author": "Thereaper29", - "publication_date": "2020-07-20T05:43:40Z", - "url": "https://www.reddit.com/r/aww/comments/hueawt/they_say_taking_breaks_is_the_key_to_productivity/", - "read": false, - "rule": 81, - "remote_identifier": "hueawt" - } -}, -{ - "model": "core.post", - "pk": 3110, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.687Z", - "title": "I went away for 3 weeks, and now my cat is in love with my husband", - "body": "
\"I
", - "author": "sillykittyish", - "publication_date": "2020-07-20T03:29:11Z", - "url": "https://www.reddit.com/r/aww/comments/hucd7u/i_went_away_for_3_weeks_and_now_my_cat_is_in_love/", - "read": false, - "rule": 81, - "remote_identifier": "hucd7u" - } -}, -{ - "model": "core.post", - "pk": 3111, - "fields": { - "created": "2020-07-20T19:32:35.636Z", - "modified": "2020-07-20T19:32:35.689Z", - "title": "Can you feel the love", - "body": "
", - "author": "kettySewrdPic", - "publication_date": "2020-07-20T09:13:32Z", - "url": "https://www.reddit.com/r/aww/comments/hugx1k/can_you_feel_the_love/", - "read": false, - "rule": 81, - "remote_identifier": "hugx1k" - } -}, -{ - "model": "core.post", - "pk": 3112, - "fields": { - "created": "2020-07-20T19:32:35.835Z", - "modified": "2020-07-21T20:14:50.522Z", - "title": "Linux Experiences/Rants or Education/Certifications thread - July 20, 2020", - "body": "

Welcome to r/linux rants and experiences! This megathread is also to hear opinions from anyone just starting out with Linux or those that have used Linux (GNU or otherwise) for a long time.

\n\n

Let us know what's annoying you, whats making you happy, or something that you want to get out to r/linux but didn't make the cut into a full post of it's own.

\n\n

For those looking for certifications please use this megathread to ask about how to get certified whether it's for the business world or for your own satisfaction. Be sure to check out r/linuxadmin for more discussion in the SysAdmin world!

\n\n

Please keep questions in r/linuxquestions, r/linux4noobs, or the Wednesday automod thread.

\n
", - "author": "AutoModerator", - "publication_date": "2020-07-20T06:12:00Z", - "url": "https://www.reddit.com/r/linux/comments/hueoo0/linux_experiencesrants_or_educationcertifications/", - "read": false, - "rule": 80, - "remote_identifier": "hueoo0" - } -}, -{ - "model": "core.post", - "pk": 3113, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:19:49.339Z", - "title": "Unix Family Tree", - "body": "
\"Unix
", - "author": "bauripalash", - "publication_date": "2020-07-20T10:32:15Z", - "url": "https://www.reddit.com/r/linux/comments/huhqrh/unix_family_tree/", - "read": true, - "rule": 80, - "remote_identifier": "huhqrh" - } -}, -{ - "model": "core.post", - "pk": 3114, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:14:50.554Z", - "title": "NVIDIA open sourced part of NVAPI SDK to aid 'Windows emulation environments'", - "body": "", - "author": "ignapk", - "publication_date": "2020-07-20T13:17:19Z", - "url": "https://www.reddit.com/r/linux/comments/huji8c/nvidia_open_sourced_part_of_nvapi_sdk_to_aid/", - "read": false, - "rule": 80, - "remote_identifier": "huji8c" - } -}, -{ - "model": "core.post", - "pk": 3115, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:14:50.551Z", - "title": "Jellyfin 10.6 released", - "body": "", - "author": "resoluti0n_", - "publication_date": "2020-07-20T16:40:05Z", - "url": "https://www.reddit.com/r/linux/comments/humekr/jellyfin_106_released/", - "read": false, - "rule": 80, - "remote_identifier": "humekr" - } -}, -{ - "model": "core.post", - "pk": 3116, - "fields": { - "created": "2020-07-20T19:32:35.836Z", - "modified": "2020-07-21T20:14:50.583Z", - "title": "[German] Article in major german newspaper about trying Linux and WSL. Literal: \"Why it's beneficial to try Linux now\"", - "body": "", - "author": "noname7890", - "publication_date": "2020-07-19T15:19:27Z", - "url": "https://www.reddit.com/r/linux/comments/hu0d5v/german_article_in_major_german_newspaper_about/", - "read": false, - "rule": 80, - "remote_identifier": "hu0d5v" - } -}, -{ - "model": "core.post", - "pk": 3117, - "fields": { - "created": "2020-07-20T19:32:35.837Z", - "modified": "2020-07-21T20:14:50.574Z", - "title": "Brian Kernighan: UNIX, C, AWK, AMPL, and Go Programming | AI Podcast #109 with Lex Fridman", - "body": "", - "author": "tinyatom", - "publication_date": "2020-07-20T08:48:35Z", - "url": "https://www.reddit.com/r/linux/comments/hugn0w/brian_kernighan_unix_c_awk_ampl_and_go/", - "read": false, - "rule": 80, - "remote_identifier": "hugn0w" - } -}, -{ - "model": "core.post", - "pk": 3118, - "fields": { - "created": "2020-07-20T19:32:35.837Z", - "modified": "2020-07-21T20:14:50.578Z", - "title": "Explaining Computers Host Christopher Barnatt Has Switched To Linux", - "body": "", - "author": "sysrpl", - "publication_date": "2020-07-20T13:00:02Z", - "url": "https://www.reddit.com/r/linux/comments/hujb12/explaining_computers_host_christopher_barnatt_has/", - "read": false, - "rule": 80, - "remote_identifier": "hujb12" - } -}, -{ - "model": "core.post", - "pk": 3119, - "fields": { - "created": "2020-07-20T19:32:35.837Z", - "modified": "2020-07-21T20:14:50.529Z", - "title": "Ireland donates contact tracing app to the Linux foundation.", - "body": "", - "author": "mathiasryan", - "publication_date": "2020-07-20T21:31:43Z", - "url": "https://www.reddit.com/r/linux/comments/hury4e/ireland_donates_contact_tracing_app_to_the_linux/", - "read": false, - "rule": 80, - "remote_identifier": "hury4e" - } -}, -{ - "model": "core.post", - "pk": 3120, - "fields": { - "created": "2020-07-20T19:32:35.842Z", - "modified": "2020-07-21T20:14:50.588Z", - "title": "I implemented a simple terminal-based password manager", - "body": "

I created a simple, secure, and free password manager written in C: SaltPass. I haven't contributed open source code before, but I think this might be useful to a few people. Especially as an alternative to paid solutions such as LastPass and the likes. Any suggestions/edits/code improvements would be greatly appreciated!

\n
", - "author": "zaid-gg", - "publication_date": "2020-07-20T07:43:03Z", - "url": "https://www.reddit.com/r/linux/comments/hufula/i_implemented_a_simple_terminalbased_password/", - "read": false, - "rule": 80, - "remote_identifier": "hufula" - } -}, -{ - "model": "core.post", - "pk": 3121, - "fields": { - "created": "2020-07-20T19:32:35.843Z", - "modified": "2020-07-21T20:14:50.593Z", - "title": "Performance analysis of multi services on container Docker, LXC, and LXD - Bulletin of Electrical Engineering and Informatics, Adinda Riztia Putri, Rendy Munadi, Ridha Muldina Negara Adaptive Network\u2026", - "body": "", - "author": "bmullan", - "publication_date": "2020-07-20T11:35:59Z", - "url": "https://www.reddit.com/r/linux/comments/huieio/performance_analysis_of_multi_services_on/", - "read": false, - "rule": 80, - "remote_identifier": "huieio" - } -}, -{ - "model": "core.post", - "pk": 3122, - "fields": { - "created": "2020-07-20T19:32:35.844Z", - "modified": "2020-07-21T20:14:50.602Z", - "title": "Create an Internal PKI using OpenSSL and NitroKey HSM", - "body": "", - "author": "PixelPaulaus", - "publication_date": "2020-07-20T06:18:41Z", - "url": "https://www.reddit.com/r/linux/comments/huerpn/create_an_internal_pki_using_openssl_and_nitrokey/", - "read": false, - "rule": 80, - "remote_identifier": "huerpn" - } -}, -{ - "model": "core.post", - "pk": 3123, - "fields": { - "created": "2020-07-20T19:32:35.844Z", - "modified": "2020-07-20T19:32:35.883Z", - "title": "vopono - run applications via VPNs with temporary network namespaces", - "body": "", - "author": "nivenkos", - "publication_date": "2020-07-19T20:02:57Z", - "url": "https://www.reddit.com/r/linux/comments/hu4vge/vopono_run_applications_via_vpns_with_temporary/", - "read": false, - "rule": 80, - "remote_identifier": "hu4vge" - } -}, -{ - "model": "core.post", - "pk": 3124, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.886Z", - "title": "Double (triple, quadruple...) internet speed with openvpn tap channel bonding to a linux VPS", - "body": "

I have been working a couple of days on my latest video about channel bonding - the video is heavily inspired be this article on Serverfault. In essence, I have been searching for a while on how to bond multiple VPN channels together in order to increase internet speed - there does not seem to be a lot of information around - mainly articles on forums and reddit state that it should be possible but a detailed guide is hard to find. I am using two Ubuntu machines in order to build the connection - one local and one VPS. The bash scripts I use in my video in order to achieve tap channel bonding are available on my github repository. I am currently working on a second video in order to walk through and explain the scripts in depth. Enjoy!

\n\n

(EDIT) - the question has come up in the discussions below if this is really packet load balancing or rather balancing links only - please see my comment further down - I can confirm that this DOES packet balancing so it does work as described.

\n
", - "author": "onemarcfifty", - "publication_date": "2020-07-19T20:41:40Z", - "url": "https://www.reddit.com/r/linux/comments/hu5l4f/double_triple_quadruple_internet_speed_with/", - "read": false, - "rule": 80, - "remote_identifier": "hu5l4f" - } -}, -{ - "model": "core.post", - "pk": 3125, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.888Z", - "title": "OpenRGB - Open source RGB lighting control that doesn't depend on manufacturer software, supports Linux", - "body": "", - "author": "pr0_c0d3", - "publication_date": "2020-07-18T16:52:48Z", - "url": "https://www.reddit.com/r/linux/comments/hthuli/openrgb_open_source_rgb_lighting_control_that/", - "read": false, - "rule": 80, - "remote_identifier": "hthuli" - } -}, -{ - "model": "core.post", - "pk": 3126, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.890Z", - "title": "Make this any sense? Automatic CPU Speed & Power Optimizer", - "body": "", - "author": "spite77", - "publication_date": "2020-07-20T11:53:35Z", - "url": "https://www.reddit.com/r/linux/comments/huikxz/make_this_any_sense_automatic_cpu_speed_power/", - "read": false, - "rule": 80, - "remote_identifier": "huikxz" - } -}, -{ - "model": "core.post", - "pk": 3127, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.891Z", - "title": "Let\u2019s not be pedantic about \u201cOpen Source\u201d", - "body": "", - "author": "speckz", - "publication_date": "2020-07-20T16:46:43Z", - "url": "https://www.reddit.com/r/linux/comments/humirw/lets_not_be_pedantic_about_open_source/", - "read": false, - "rule": 80, - "remote_identifier": "humirw" - } -}, -{ - "model": "core.post", - "pk": 3128, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.893Z", - "title": "Experiences with running Linux Lite", - "body": "", - "author": "daemonpenguin", - "publication_date": "2020-07-20T02:43:49Z", - "url": "https://www.reddit.com/r/linux/comments/hubonw/experiences_with_running_linux_lite/", - "read": false, - "rule": 80, - "remote_identifier": "hubonw" - } -}, -{ - "model": "core.post", - "pk": 3129, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.895Z", - "title": "Tried gnome on arch, surprised how lean it is (used flameshot so it used about 72mb more) closing at 600 megs) on fedora and pop i had gnome eating up 1.3gigs at boot up.", - "body": "
\"Tried
", - "author": "V1n0dKr1shna", - "publication_date": "2020-07-18T13:54:55Z", - "url": "https://www.reddit.com/r/linux/comments/htfeph/tried_gnome_on_arch_surprised_how_lean_it_is_used/", - "read": false, - "rule": 80, - "remote_identifier": "htfeph" - } -}, -{ - "model": "core.post", - "pk": 3130, - "fields": { - "created": "2020-07-20T19:32:35.849Z", - "modified": "2020-07-20T19:32:35.897Z", - "title": "The Free Software Foundation is holding a Fundraiser, help them reach 200 members", - "body": "", - "author": "Neet-Feet", - "publication_date": "2020-07-18T17:55:30Z", - "url": "https://www.reddit.com/r/linux/comments/htiuyi/the_free_software_foundation_is_holding_a/", - "read": false, - "rule": 80, - "remote_identifier": "htiuyi" - } -}, -{ - "model": "core.post", - "pk": 3131, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.899Z", - "title": "Why is the mindset around Arch so negative?", - "body": "

I love the Linux community as a whole. You can find some of the most creative and imaginative people within most Linux communities. On a whole, Linux users are some of the most helpful and informative people you can encounter. Truly the type to think outside the box and learn new things. It can be very inspirational.

\n\n

If I jumped onto Ubuntu, Fedora, or openSUSE's community I can have a free flowing conversation about Linux, their distribution, and getting help or giving help is so free-flowing and easy. The communities are eager to welcome new people and appreciate folks who contribute.

\n\n

Then you have Arch. I love the OS but dislike the mindset. Asking for help is meat with resistance, giving help can also be punishable, and god forbid you try to have a discussion. But it's not just their core community either. For example, I just discovered Endeavour OS which is built around Arch and after 11 post I'm told to come back in 8 hours. Their subReddit here on Reddit, you have to ask to even make 1 post. There of course is also Manjaro Linux and they too have this gatekeeper mindset, the same can be said for ArcoLinux.

\n\n

What is it about Arch that makes everyone want to be either a control freak or a gatekeeper?

\n\n

I do not see this within the Ubuntu or Fedora or openSUSE communities. As I said, their mindset seems eager and willing to unite and work as a community. Am I the only how has noticed this?

\n
", - "author": "Linux-Is-Best", - "publication_date": "2020-07-18T23:28:12Z", - "url": "https://www.reddit.com/r/linux/comments/htojwk/why_is_the_mindset_around_arch_so_negative/", - "read": false, - "rule": 80, - "remote_identifier": "htojwk" - } -}, -{ - "model": "core.post", - "pk": 3132, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.901Z", - "title": "Using the nstat network statistics command in Linux", - "body": "", - "author": "cronos426", - "publication_date": "2020-07-19T17:55:55Z", - "url": "https://www.reddit.com/r/linux/comments/hu2q6v/using_the_nstat_network_statistics_command_in/", - "read": false, - "rule": 80, - "remote_identifier": "hu2q6v" - } -}, -{ - "model": "core.post", - "pk": 3133, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.903Z", - "title": "Contributing via GitLab Merge Requests", - "body": "", - "author": "ChristophCullmann", - "publication_date": "2020-07-18T20:01:26Z", - "url": "https://www.reddit.com/r/linux/comments/htl05p/contributing_via_gitlab_merge_requests/", - "read": false, - "rule": 80, - "remote_identifier": "htl05p" - } -}, -{ - "model": "core.post", - "pk": 3134, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.905Z", - "title": "OpenMandriva: combines WINE64 and 32 into one package capable of running both binaries, i686 architecture was considered as deprecated. Work is underway on a new Rolling release", - "body": "", - "author": "DamonsLinux", - "publication_date": "2020-07-18T15:02:35Z", - "url": "https://www.reddit.com/r/linux/comments/htg9dj/openmandriva_combines_wine64_and_32_into_one/", - "read": false, - "rule": 80, - "remote_identifier": "htg9dj" - } -}, -{ - "model": "core.post", - "pk": 3135, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.906Z", - "title": "OpenRCT2 Player Survey 2020 - Previous survey shows almost 25% players are linux, please help represent linux in the most recent survey", - "body": "", - "author": "christophski", - "publication_date": "2020-07-18T11:39:06Z", - "url": "https://www.reddit.com/r/linux/comments/htdzuh/openrct2_player_survey_2020_previous_survey_shows/", - "read": false, - "rule": 80, - "remote_identifier": "htdzuh" - } -}, -{ - "model": "core.post", - "pk": 3136, - "fields": { - "created": "2020-07-20T19:32:35.853Z", - "modified": "2020-07-20T19:32:35.908Z", - "title": "This week in KDE: Get New Stuff fixes and more", - "body": "", - "author": "kyentei", - "publication_date": "2020-07-18T10:03:46Z", - "url": "https://www.reddit.com/r/linux/comments/htd1an/this_week_in_kde_get_new_stuff_fixes_and_more/", - "read": false, - "rule": 80, - "remote_identifier": "htd1an" - } -}, -{ - "model": "core.post", - "pk": 3137, - "fields": { - "created": "2020-07-20T19:32:35.857Z", - "modified": "2020-07-20T19:32:35.910Z", - "title": "Blender Runs on Linux Pinephone", - "body": "

I managed to get the desktop version of Blender on the Pinephone, and it works really well except for a few bugs.

\n\n

See my post on r/blender:

\n\n

https://www.reddit.com/r/blender/comments/hsxv27/i_installed_blender_on_a_phone/

\n\n

and r/PINE64official:

\n\n

https://www.reddit.com/r/PINE64official/comments/hsxc33/blender_on_pine_phone_almost_usable/

\n\n

I've tried other desktop programs like Xournal and PPSSPP, their UIs also work well, I'd be able to do even more if OpenGL 3 was working.

\n
", - "author": "InfiniteHawk", - "publication_date": "2020-07-17T22:35:14Z", - "url": "https://www.reddit.com/r/linux/comments/ht3d4k/blender_runs_on_linux_pinephone/", - "read": false, - "rule": 80, - "remote_identifier": "ht3d4k" - } -}, -{ - "model": "core.post", - "pk": 3138, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:18:21.616Z", - "title": "Hrmmm They Need to Fix Throttle Animations in the Sabre", - "body": "
", - "author": "TheBootRanger", - "publication_date": "2020-07-21T13:26:01Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv5omc/hrmmm_they_need_to_fix_throttle_animations_in_the/", - "read": true, - "rule": 82, - "remote_identifier": "hv5omc" - } -}, -{ - "model": "core.post", - "pk": 3139, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:18:49.999Z", - "title": "My first 3.10 landing could have gone better...", - "body": "
", - "author": "KnLfey", - "publication_date": "2020-07-21T16:04:50Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv7w85/my_first_310_landing_could_have_gone_better/", - "read": true, - "rule": 82, - "remote_identifier": "hv7w85" - } -}, -{ - "model": "core.post", - "pk": 3140, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:14:50.439Z", - "title": "How about the Christmas in 3 more years?", - "body": "
\"How
", - "author": "SpleanEater", - "publication_date": "2020-07-21T17:49:22Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv9qy8/how_about_the_christmas_in_3_more_years/", - "read": false, - "rule": 82, - "remote_identifier": "hv9qy8" - } -}, -{ - "model": "core.post", - "pk": 3141, - "fields": { - "created": "2020-07-21T20:14:50.415Z", - "modified": "2020-07-21T20:18:33.532Z", - "title": "Long time Elite Dangerous player. New to star citizen i think im doing great", - "body": "", - "author": "Filblo5", - "publication_date": "2020-07-21T15:33:49Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv7elb/long_time_elite_dangerous_player_new_to_star/", - "read": true, - "rule": 82, - "remote_identifier": "hv7elb" - } -}, -{ - "model": "core.post", - "pk": 3142, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.443Z", - "title": "And we stand by it.", - "body": "
\"And
", - "author": "CyberTill", - "publication_date": "2020-07-21T18:57:48Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvb3wm/and_we_stand_by_it/", - "read": false, - "rule": 82, - "remote_identifier": "hvb3wm" - } -}, -{ - "model": "core.post", - "pk": 3143, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.446Z", - "title": "Nomad", - "body": "
\"Nomad\"
", - "author": "ibracitizen", - "publication_date": "2020-07-21T19:52:24Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvc5h3/nomad/", - "read": false, - "rule": 82, - "remote_identifier": "hvc5h3" - } -}, -{ - "model": "core.post", - "pk": 3144, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.449Z", - "title": "Probably the best screen cap i've ever caught on a whim. 3.5 Arc Corp release. Also a confession: I never pledged. Got a ship with my GPU. I intend to pay my dues.", - "body": "
\"Probably
", - "author": "ScionoicS", - "publication_date": "2020-07-21T20:23:01Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvcqzf/probably_the_best_screen_cap_ive_ever_caught_on_a/", - "read": false, - "rule": 82, - "remote_identifier": "hvcqzf" - } -}, -{ - "model": "core.post", - "pk": 3145, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:14:50.451Z", - "title": "Play to escape the depressing job hunt where I need 10 years experience for a entry level job to find this, only been playing for 1 and a half years :(", - "body": "
\"Play
", - "author": "Albert-III-", - "publication_date": "2020-07-21T12:23:45Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv4z08/play_to_escape_the_depressing_job_hunt_where_i/", - "read": false, - "rule": 82, - "remote_identifier": "hv4z08" - } -}, -{ - "model": "core.post", - "pk": 3146, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:19:00.691Z", - "title": "The void beckons.", - "body": "
", - "author": "HisNameWasHis", - "publication_date": "2020-07-21T14:40:51Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv6nij/the_void_beckons/", - "read": true, - "rule": 82, - "remote_identifier": "hv6nij" - } -}, -{ - "model": "core.post", - "pk": 3147, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:19:05.881Z", - "title": "I made a SC-like Photobash with Soldiers", - "body": "
\"I
", - "author": "IsaacPolar", - "publication_date": "2020-07-21T17:13:39Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv92ri/i_made_a_sclike_photobash_with_soldiers/", - "read": true, - "rule": 82, - "remote_identifier": "hv92ri" - } -}, -{ - "model": "core.post", - "pk": 3148, - "fields": { - "created": "2020-07-21T20:14:50.416Z", - "modified": "2020-07-21T20:19:41.227Z", - "title": "Ocean Shader Improvements", - "body": "
\"Ocean
", - "author": "shoeii", - "publication_date": "2020-07-21T18:41:51Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvasds/ocean_shader_improvements/", - "read": true, - "rule": 82, - "remote_identifier": "hvasds" - } -}, -{ - "model": "core.post", - "pk": 3149, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.459Z", - "title": "As much shit as Star Citizen (rightfully) gets it still does one thing better than any other 'game' I've played", - "body": "

It invokes a real sense of scale, on multiple levels.

\n\n

One could argue that's one of the most important feelings you'd want to capture in any game set in space, but of course it's mostly meaningless if there aren't enough gameplay loops and systems in place to work in tandem with and make the space that's been created interesting, and that's where SC is currently a failure.

\n\n

Even so, I think being able to create that sense of smallness isn't insignificant.

\n\n

You as a pilot are dwarfed by your ship which is itself dwarfed by a larger ship which is itself dwarfed by another, even more massive one which is dwarfed by the space station or hub you're at which is dwarfed by a crater on a moon which is dwarfed by the moon itself which is dwarfed by the planet it orbits which is dwarfed by the sheer vastness of space in between all of those things and that they are, despite the distance, still connected.

\n\n

Getting lost in Lorville (even if it is mostly linear) and knowing it's only a small part of the playable space is a really neat feeling - looking out from the windows of the train up into the sky and knowing you can go there and beyond really makes you feel like there is a whole world (and more) waiting to be explored.

\n\n

I think this is a direct result of having legs and not being locked into the cockpit of your ship - I've played more Elite: Dangerous than Star Citizen and it accomplishes a similar sense of scale but, at least not as far as I've felt, never to the same degree - because you're locked in your ship you never really get this same sense of being small or insignificant even though you are dwarfed in similar ways by planets/asteroids/other ships - will be interesting to see how their implementation of 'space legs' in the upcoming expansion changes this.

\n\n

My favourite thing to do in Star Citizen (because there isn't a whole lot) is to just find some pocket of space far away from anything else and just walk around my ship, feeling truly alone and insignificant, gazing out at the void that stretches infinitely all around - something about that is super comfy.

\n\n

I can't think of many other game that accomplish a similar level of scale though I'm sure they exist.

\n\n

I've been playing an indie game called Empyrion - Galactic Survival and it actually is sort of similar to SC in this regard but it's nowhere near as polished or smooth - transitions from atmosphere to space are not truly seamless and planets themselves are kind of stitched together, but it still manages to invoke that same kind of awe at the scale of things when you dock a small vessel to a capital vessel, for example - definitely worth checking out if you like sci-fi/space games, which you must if you're here, but just be prepared for the jank.

\n
", - "author": "thegreatself", - "publication_date": "2020-07-21T20:30:15Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvcw38/as_much_shit_as_star_citizen_rightfully_gets_it/", - "read": false, - "rule": 82, - "remote_identifier": "hvcw38" - } -}, -{ - "model": "core.post", - "pk": 3150, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.462Z", - "title": "You waiting for patch 3.10 to go live while watching tons of videos about the new flight model features. Be patient, 3.11 and 3.12 will be even better.", - "body": "
\"You
", - "author": "jsabater76", - "publication_date": "2020-07-21T09:39:27Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv372v/you_waiting_for_patch_310_to_go_live_while/", - "read": false, - "rule": 82, - "remote_identifier": "hv372v" - } -}, -{ - "model": "core.post", - "pk": 3151, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.466Z", - "title": "CIG, can we please fix these \"black hole\" doors(when they are closed) on ships please.", - "body": "
\"CIG,
", - "author": "AbnormallyBendPenis", - "publication_date": "2020-07-21T13:40:14Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv5uzj/cig_can_we_please_fix_these_black_hole_doorswhen/", - "read": false, - "rule": 82, - "remote_identifier": "hv5uzj" - } -}, -{ - "model": "core.post", - "pk": 3152, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.468Z", - "title": "Anvil Super Hornet over Cellin", - "body": "
\"Anvil
", - "author": "SaraCaterina", - "publication_date": "2020-07-21T20:33:58Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvcyq6/anvil_super_hornet_over_cellin/", - "read": false, - "rule": 82, - "remote_identifier": "hvcyq6" - } -}, -{ - "model": "core.post", - "pk": 3153, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.471Z", - "title": "3.10 Combat Changes", - "body": "", - "author": "STLYoungblood", - "publication_date": "2020-07-21T16:37:44Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv8fr7/310_combat_changes/", - "read": false, - "rule": 82, - "remote_identifier": "hv8fr7" - } -}, -{ - "model": "core.post", - "pk": 3154, - "fields": { - "created": "2020-07-21T20:14:50.420Z", - "modified": "2020-07-21T20:14:50.472Z", - "title": "Hey CIG how about that S42 Vi.... Oh...", - "body": "
\"Hey
", - "author": "SiEDeN", - "publication_date": "2020-07-21T21:37:16Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hve6am/hey_cig_how_about_that_s42_vi_oh/", - "read": false, - "rule": 82, - "remote_identifier": "hve6am" - } -}, -{ - "model": "core.post", - "pk": 3155, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.475Z", - "title": "3.10 M PTU Eclipse improvements", - "body": "

If this goes live, CIG had addressed 2 of my Eclipse critics.

\n\n

Not because of my videos of course, CIG doesn't know I exist.

\n\n

 

\n\n

a. Eclipse has armor stealth in 3.10, see my table:\nhttps://docs.google.com/spreadsheets/d/1OJXg7MQsG_IVTPsmlmZYaxEPK4n4iqnhQx4oigIlJHg/edit#gid=343807746

\n\n

 

\n\n

b. Eclipse can fire her size 9 torpedoes way quicker now, see my video with a side by side comparison of the max firing speed in 3.9 and 3.10:\nhttps://youtu.be/GFTF1Qt7T3o?t=207

\n
", - "author": "Camural", - "publication_date": "2020-07-21T18:15:50Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hva9lc/310_m_ptu_eclipse_improvements/", - "read": false, - "rule": 82, - "remote_identifier": "hva9lc" - } -}, -{ - "model": "core.post", - "pk": 3156, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.477Z", - "title": "Hark! The Drake Herald Sings", - "body": "
\"Hark!
", - "author": "CyrexStorm", - "publication_date": "2020-07-21T16:19:31Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv84kk/hark_the_drake_herald_sings/", - "read": false, - "rule": 82, - "remote_identifier": "hv84kk" - } -}, -{ - "model": "core.post", - "pk": 3157, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.479Z", - "title": "The new flight stick in the Prowler", - "body": "
\"The
", - "author": "Potato_Nades", - "publication_date": "2020-07-21T16:22:22Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv86c2/the_new_flight_stick_in_the_prowler/", - "read": false, - "rule": 82, - "remote_identifier": "hv86c2" - } -}, -{ - "model": "core.post", - "pk": 3158, - "fields": { - "created": "2020-07-21T20:14:50.422Z", - "modified": "2020-07-21T20:14:50.481Z", - "title": "Norwegian VAT charged from August 1st", - "body": "
\"Norwegian
", - "author": "norgeek", - "publication_date": "2020-07-21T10:30:57Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv3r3l/norwegian_vat_charged_from_august_1st/", - "read": false, - "rule": 82, - "remote_identifier": "hv3r3l" - } -}, -{ - "model": "core.post", - "pk": 3159, - "fields": { - "created": "2020-07-21T20:14:50.423Z", - "modified": "2020-07-21T20:14:50.484Z", - "title": "With Pyro (currently WIP), Nyx (partially done), Odin (S42), currently on the way, what is everyone\u2019s thoughts on Terra possibly being next on the list of star systems to be added into the PU within \u2026", - "body": "
\"With
", - "author": "realCLTotaku", - "publication_date": "2020-07-21T13:27:09Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hv5p41/with_pyro_currently_wip_nyx_partially_done_odin/", - "read": false, - "rule": 82, - "remote_identifier": "hv5p41" - } -}, -{ - "model": "core.post", - "pk": 3160, - "fields": { - "created": "2020-07-21T20:14:50.423Z", - "modified": "2020-07-21T20:14:50.486Z", - "title": "Testing out the new electron rifle", - "body": "
", - "author": "joshbaker2112", - "publication_date": "2020-07-21T02:56:19Z", - "url": "https://www.reddit.com/r/starcitizen/comments/huxr6d/testing_out_the_new_electron_rifle/", - "read": false, - "rule": 82, - "remote_identifier": "huxr6d" - } -}, -{ - "model": "core.post", - "pk": 3161, - "fields": { - "created": "2020-07-21T20:14:50.423Z", - "modified": "2020-07-21T20:14:50.487Z", - "title": "Imperial Geographic's Lovecraftian magazine special is here. \ud83d\udc19 Find the link in the comments!", - "body": "
\"Imperial
", - "author": "Good_Punk2", - "publication_date": "2020-07-21T18:21:38Z", - "url": "https://www.reddit.com/r/starcitizen/comments/hvadrh/imperial_geographics_lovecraftian_magazine/", - "read": false, - "rule": 82, - "remote_identifier": "hvadrh" - } -}, -{ - "model": "core.post", - "pk": 3162, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.525Z", - "title": "Linux Distributions Timeline", - "body": "
\"Linux
", - "author": "bauripalash", - "publication_date": "2020-07-21T06:07:59Z", - "url": "https://www.reddit.com/r/linux/comments/hv0ktn/linux_distributions_timeline/", - "read": false, - "rule": 80, - "remote_identifier": "hv0ktn" - } -}, -{ - "model": "core.post", - "pk": 3163, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.527Z", - "title": "Fedora: Proposal to replace default wined3d backend with DXVK", - "body": "", - "author": "friskfrugt", - "publication_date": "2020-07-21T19:42:49Z", - "url": "https://www.reddit.com/r/linux/comments/hvbyyr/fedora_proposal_to_replace_default_wined3d/", - "read": false, - "rule": 80, - "remote_identifier": "hvbyyr" - } -}, -{ - "model": "core.post", - "pk": 3164, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.531Z", - "title": "Update on marketing and communication plans for the LibreOffice 7.x series", - "body": "", - "author": "TheQuantumZero", - "publication_date": "2020-07-21T09:59:23Z", - "url": "https://www.reddit.com/r/linux/comments/hv3erm/update_on_marketing_and_communication_plans_for/", - "read": false, - "rule": 80, - "remote_identifier": "hv3erm" - } -}, -{ - "model": "core.post", - "pk": 3165, - "fields": { - "created": "2020-07-21T20:14:50.497Z", - "modified": "2020-07-21T20:14:50.533Z", - "title": "FOSS job opening: LibreOffice Development Mentor at The Document Foundation", - "body": "", - "author": "themikeosguy", - "publication_date": "2020-07-21T14:26:36Z", - "url": "https://www.reddit.com/r/linux/comments/hv6gfw/foss_job_opening_libreoffice_development_mentor/", - "read": false, - "rule": 80, - "remote_identifier": "hv6gfw" - } -}, -{ - "model": "core.post", - "pk": 3166, - "fields": { - "created": "2020-07-21T20:14:50.503Z", - "modified": "2020-07-21T20:14:50.536Z", - "title": "gomd - quickly display formatted markdown files with code highlight in your browser", - "body": "

Hi all!

\n\n

I wanted to share a project I've been working on recently. I think it reached a stage where it's pretty usable and should work out of the box. gomd sets up a HTTP server and serves a directory in your browser so you can quickly view your markdown files. It comes with some neat features like:

\n\n
    \n
  • Monitoring files - it will monitor files for changes and reload them whenever needed
    • \n
  • Hot reloading - whenever the file you are currently viewing changes, the tab in your browser will reload automatically.
    • \n
  • Code Highlight - All blocks of code in most common languages will be color highlighted.
    • \n
  • Themes - choose from multiple themes like: solarized, monokai, github, dracula...
    • \n
\n\n

Link: gomd

\n\n

For now its only available from AUR or built from source.

\n\n

​

\n\n

Any tips or feedback will be greatly appreciated :)

\n
", - "author": "wwojtekk", - "publication_date": "2020-07-21T20:07:31Z", - "url": "https://www.reddit.com/r/linux/comments/hvcg44/gomd_quickly_display_formatted_markdown_files/", - "read": false, - "rule": 80, - "remote_identifier": "hvcg44" - } -}, -{ - "model": "core.post", - "pk": 3167, - "fields": { - "created": "2020-07-21T20:14:50.503Z", - "modified": "2020-07-21T20:14:50.543Z", - "title": "They're not otherwise wrong, but it didn't become a real Internet standard until 2017.", - "body": "
\"They're
", - "author": "foodown", - "publication_date": "2020-07-21T21:39:09Z", - "url": "https://www.reddit.com/r/linux/comments/hve7l5/theyre_not_otherwise_wrong_but_it_didnt_become_a/", - "read": false, - "rule": 80, - "remote_identifier": "hve7l5" - } -}, -{ - "model": "core.post", - "pk": 3168, - "fields": { - "created": "2020-07-21T20:14:50.503Z", - "modified": "2020-07-21T20:14:50.545Z", - "title": "Drawing - an alternative to Paint for Linux (gtk3, support HiDPI)", - "body": "", - "author": "dontdieych", - "publication_date": "2020-07-21T02:37:22Z", - "url": "https://www.reddit.com/r/linux/comments/huxgsg/drawing_an_alternative_to_paint_for_linux_gtk3/", - "read": false, - "rule": 80, - "remote_identifier": "huxgsg" - } -}, -{ - "model": "core.post", - "pk": 3169, - "fields": { - "created": "2020-07-21T20:14:50.509Z", - "modified": "2020-07-21T20:14:50.547Z", - "title": "Observations on a Linux issue with 3.5mm earphones with a mic", - "body": "

Alright hello. I have come from r/SolusProject and I made a post there to do with headphone issues. I suggest you read through the post and comments to get a better understanding before reading this https://www.reddit.com/r/SolusProject/comments/hsql4d/frustrating_headphone_issues/. I had posted to do with it again, but it got taken down for duplication (when it wasn't duplication). This post is more of my observations from experimenting and such. There are distros I haven't tried but I tried a wide range of distros like manjaro, ubuntu based ones and all solus flavors, and I was looking more for how well they worked out of the box, rather than with fiddling around with pulse, hdajack etc which I know will work eventually. If you stumbled across this from searching about the same issue I have (or similar) or are confused to what this is about, I suggest you look at my previous post also.

\n\n

So anyways, I've tried the past few days mounting isos to usb drives and trying live os and installing various distros to see about the headphone issue. And my conclusion is that this issue affects the linux kernel in some way across the board. I don't really understand why completely but I have some kind of idea.

\n\n

From installing fresh distros, I noticed that the earphones (they are 3.5mm earphones + mic) get recognised as a microphone and not as a speaker system of some kind. Every single time I had a look at the sound settings and in pulse, they came up as plugged microphone, with the internal speakers being the only output device every single time. It's really odd seeing as how ubuntu 14.04 and xubuntu etc from years past worked flawlessly with the earphones, even manjaro a while ago on my older craptop worked fine. I don't really understand why it doesn't work on my device now.

\n\n

I'll leave my specs at the bottom of this post but what I think is is there's something the manufacturer did, or something like the cpu causes issue with linux. The manufacturer of my laptop is Lenovo, and the cpu/igpu is from AMD. A warning sign is that when installing a linux distro, it doesn't bring up the dual boot menu at startup like it should. Instead it completely hides the fact it exists until I use something like easyuefi to add an option for that distro, how it works is you specify the boot partition, whether it's linux or windows and the loader conf file for the distro. All of this hassle everytime doesn't appear on my craptop, because the dual boot menu appears flawlessly without issue. May be because it uses an Intel cpu/igpu unlike my newer laptop but it's hard to say.

\n\n

Also, it seems like the devices that appear in a given distro when looking at alsa, is hd generic devices but by reloading alsa or any command that shows the full name of the device, it says it's Intel. I don't know if that would be an issue, maybe amd use intel sound drivers or something. It's odd nonetheless.

\n\n

This issue has been boggling my mind for obvious reasons, with half-rhetorical questions like does linux not support the earphones anymore, whether out of accident from an overlooked bug in an update or intentionally phasing out? Is any of this AMD or Lenovo's fault? Even with proper headphones or something, will they fail? I don't think anyone here really knows, hell I'd bet an extreme that no one really understands why in the linux community. I kinda rambled in this post with stuff that should've been said in the last post/thread, but I'm saying it now.

\n\n

Thanks for contributing thus far to this discussion in figuring this out.

\n\n

Specs: AMD Ryzen 5 3500U Mobile CPU (2.2 - 3.7ghz quad core)

\n\n

Radeon Vega 8 Integrated GPU, 8GB Ram, 256GB SSD.

\n\n

Lenovo C340-14API Laptop

\n
", - "author": "BrianMeerkatlol", - "publication_date": "2020-07-21T21:02:19Z", - "url": "https://www.reddit.com/r/linux/comments/hvdi3o/observations_on_a_linux_issue_with_35mm_earphones/", - "read": false, - "rule": 80, - "remote_identifier": "hvdi3o" - } -}, -{ - "model": "core.post", - "pk": 3170, - "fields": { - "created": "2020-07-21T20:14:50.509Z", - "modified": "2020-07-21T20:14:50.549Z", - "title": "South Korean distro HamoniKR OS has been added to Distrowatch", - "body": "", - "author": "TheHordeRisesAgain", - "publication_date": "2020-07-21T07:44:21Z", - "url": "https://www.reddit.com/r/linux/comments/hv1ug1/south_korean_distro_hamonikr_os_has_been_added_to/", - "read": false, - "rule": 80, - "remote_identifier": "hv1ug1" - } -}, -{ - "model": "core.post", - "pk": 3171, - "fields": { - "created": "2020-07-21T20:14:50.509Z", - "modified": "2020-07-21T20:14:50.559Z", - "title": "The Jailer is free! New release of the outstanding database subsetter and browser is available.", - "body": "", - "author": "Plane-Discussion", - "publication_date": "2020-07-21T12:53:54Z", - "url": "https://www.reddit.com/r/linux/comments/hv5b0j/the_jailer_is_free_new_release_of_the_outstanding/", - "read": false, - "rule": 80, - "remote_identifier": "hv5b0j" - } -}, -{ - "model": "core.post", - "pk": 3172, - "fields": { - "created": "2020-07-21T20:14:50.513Z", - "modified": "2020-07-21T20:14:50.563Z", - "title": "A few very well-aged excerpts from Microsoft\u2019s infamous 2004 \u201cGet the facts\u201d campaign, where they make the case for Windows servers being cheaper, more secure, and more performant than Linux servers", - "body": "
\n

Get the facts on Windows and Linux.

\n\n

Leading companies and third-party analysts confirm it: Windows has a lower total cost of ownership and outperforms Linux.

\n\n

...

\n\n

-Security

\n\n

Windows Users Have Fewer Vulnerabilities

\n
\n\n

And then literally the very next bullet point:

\n\n
\n

-Featured Customer Case Study

\n\n

Equifax

\n\n

Equifax Sees 14 Percent Cost Savings

\n\n

Find out why Equifax, a global leader in transforming data into intelligence, selected Windows over Linux to enhance the speed and performance of its marketing services capabilities. Using Microsoft Windows Server System, the company has seen 14 percent in cost savings over Linux.

\n
\n\n

Good thing they saved 14% and got all that extra security! Sure their website is janky and their login flow is downright horrifying (Check it out if you want to be amazed), but who could blame them? Linux is \u201cProhibitively Expensive, Extremely Complex, and Provides No Tangible Business Gains\u201d, Microsoft said so!

\n\n

Source: https://web.archive.org/web/20041027003759/http://www.microsoft.com/windowsserversystem/facts/default.mspx

\n
", - "author": "kevinhaze", - "publication_date": "2020-07-20T21:42:15Z", - "url": "https://www.reddit.com/r/linux/comments/hus5lz/a_few_very_wellaged_excerpts_from_microsofts/", - "read": false, - "rule": 80, - "remote_identifier": "hus5lz" - } -}, -{ - "model": "core.post", - "pk": 3173, - "fields": { - "created": "2020-07-21T20:14:50.515Z", - "modified": "2020-07-21T20:14:50.566Z", - "title": "Are there are any professional audio recording studios or artists that use Linux?", - "body": "

As the title says, who is using Linux as a professional audio engineer, producer, or artist? I am a former Mac user myself, and I am seeing people from time to time who have become disillusioned with what Apple has been doing for the past few years. However, I'm not sure if Linux really has a place for these people to land if they are serious about what they do.

\n\n

Fedora Design Suite and Ubuntu Studio are definitely encouraging to see, but what is their real-world usage like? Are we getting better with professional audio in Linux, or have things been stagnant for years?

\n
", - "author": "RootHouston", - "publication_date": "2020-07-21T00:08:26Z", - "url": "https://www.reddit.com/r/linux/comments/huuxvq/are_there_are_any_professional_audio_recording/", - "read": false, - "rule": 80, - "remote_identifier": "huuxvq" - } -}, -{ - "model": "core.post", - "pk": 3174, - "fields": { - "created": "2020-07-21T20:14:50.515Z", - "modified": "2020-07-21T20:14:50.570Z", - "title": "When Linux had marketing", - "body": "", - "author": "Commodore256", - "publication_date": "2020-07-21T14:03:56Z", - "url": "https://www.reddit.com/r/linux/comments/hv65oa/when_linux_had_marketing/", - "read": false, - "rule": 80, - "remote_identifier": "hv65oa" - } -}, -{ - "model": "core.post", - "pk": 3175, - "fields": { - "created": "2020-07-21T20:14:50.520Z", - "modified": "2020-07-21T20:14:50.598Z", - "title": "Ward: Simple and minimalistic server dashboard", - "body": "

Ward is a simple and and minimalistic server monitoring tool. Ward supports adaptive design system. Also it supports dark theme. It shows only principal information and can be used, if you want to see nice looking dashboard instead looking on bunch of numbers and graphs. Ward works nice on all popular operating systems, because it uses OSHI.

\n\n

https://preview.redd.it/gdppswc3a3c51.png?width=1448&format=png&auto=webp&s=0d6e10146c105ddcfd045dd59c970d4c127ddb8c

\n\n

https://github.com/B-Software/Ward

\n
", - "author": "Pabyzu", - "publication_date": "2020-07-21T00:33:40Z", - "url": "https://www.reddit.com/r/linux/comments/huvea3/ward_simple_and_minimalistic_server_dashboard/", - "read": false, - "rule": 80, - "remote_identifier": "huvea3" - } -}, -{ - "model": "core.post", - "pk": 3176, - "fields": { - "created": "2020-07-21T20:14:50.522Z", - "modified": "2020-07-21T20:14:50.606Z", - "title": "WindowsFX - a good Windows alternative?", - "body": "

I would personally like to hear some of your opinions (in the replies) about WindowsFX. What is WindowsFX you may ask? WindowsFX is a Brazilian linux distribution that is designed to look and act like Windows 10.

\n\n

Linux / WindowsFX is based off of Ubuntu, and uses Cinnamon as its DE. Upon first boot, normal Windows users can tell the difference. But if you were to put it in front of a non tech-savvy person, they wouldn't be able to tell the difference.

\n\n

Personally, with WSL on Windows, I see no need for a distro like this. However, as I said, I would like to hear your opinions on this distro.

\n\n

Video review here.

\n
", - "author": "Demonitized101", - "publication_date": "2020-07-20T23:03:29Z", - "url": "https://www.reddit.com/r/linux/comments/hutpt5/windowsfx_a_good_windows_alternative/", - "read": false, - "rule": 80, - "remote_identifier": "hutpt5" - } -}, -{ - "model": "core.post", - "pk": 3177, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:14:50.780Z", - "title": "Every day this good boy brings a carrot to his best buddy", - "body": "
", - "author": "TooShiftyForYou", - "publication_date": "2020-07-21T15:25:31Z", - "url": "https://www.reddit.com/r/aww/comments/hv7a8b/every_day_this_good_boy_brings_a_carrot_to_his/", - "read": false, - "rule": 81, - "remote_identifier": "hv7a8b" - } -}, -{ - "model": "core.post", - "pk": 3178, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-25T20:08:34.264Z", - "title": "Kitten mimics his human petting the dog", - "body": "
", - "author": "SpecterAscendant", - "publication_date": "2020-07-21T14:56:57Z", - "url": "https://www.reddit.com/r/aww/comments/hv6ve3/kitten_mimics_his_human_petting_the_dog/", - "read": true, - "rule": 81, - "remote_identifier": "hv6ve3" - } -}, -{ - "model": "core.post", - "pk": 3179, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:14:50.789Z", - "title": "My fox friend!", - "body": "
", - "author": "Zepantha", - "publication_date": "2020-07-21T14:27:25Z", - "url": "https://www.reddit.com/r/aww/comments/hv6gte/my_fox_friend/", - "read": false, - "rule": 81, - "remote_identifier": "hv6gte" - } -}, -{ - "model": "core.post", - "pk": 3180, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:15:46.876Z", - "title": "Ducks annihilate peas", - "body": "
", - "author": "tommycalibre", - "publication_date": "2020-07-21T17:12:40Z", - "url": "https://www.reddit.com/r/aww/comments/hv9258/ducks_annihilate_peas/", - "read": true, - "rule": 81, - "remote_identifier": "hv9258" - } -}, -{ - "model": "core.post", - "pk": 3181, - "fields": { - "created": "2020-07-21T20:14:50.775Z", - "modified": "2020-07-21T20:14:50.797Z", - "title": "Wiggle it baby", - "body": "
", - "author": "neo_star", - "publication_date": "2020-07-21T18:44:31Z", - "url": "https://www.reddit.com/r/aww/comments/hvaucy/wiggle_it_baby/", - "read": false, - "rule": 81, - "remote_identifier": "hvaucy" - } -}, -{ - "model": "core.post", - "pk": 3182, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:16:22.725Z", - "title": "I guess I should do this.. everyone seems to be liking little pups and kittens so.. Reddit, meet bailey", - "body": "
\"I
", - "author": "X_XNOTHINGX_X", - "publication_date": "2020-07-21T14:15:08Z", - "url": "https://www.reddit.com/r/aww/comments/hv6b0a/i_guess_i_should_do_this_everyone_seems_to_be/", - "read": true, - "rule": 81, - "remote_identifier": "hv6b0a" - } -}, -{ - "model": "core.post", - "pk": 3183, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.806Z", - "title": "The hat makes the crab.", - "body": "
\"The
", - "author": "fujfuj", - "publication_date": "2020-07-21T14:48:40Z", - "url": "https://www.reddit.com/r/aww/comments/hv6rde/the_hat_makes_the_crab/", - "read": false, - "rule": 81, - "remote_identifier": "hv6rde" - } -}, -{ - "model": "core.post", - "pk": 3184, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.812Z", - "title": "Baby bunny fits in hand", - "body": "
", - "author": "Hawken10", - "publication_date": "2020-07-21T12:31:30Z", - "url": "https://www.reddit.com/r/aww/comments/hv5253/baby_bunny_fits_in_hand/", - "read": false, - "rule": 81, - "remote_identifier": "hv5253" - } -}, -{ - "model": "core.post", - "pk": 3185, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.818Z", - "title": "My cat and I, both pregnant", - "body": "
\"My
", - "author": "nixdionisio", - "publication_date": "2020-07-21T11:06:25Z", - "url": "https://www.reddit.com/r/aww/comments/hv44m2/my_cat_and_i_both_pregnant/", - "read": false, - "rule": 81, - "remote_identifier": "hv44m2" - } -}, -{ - "model": "core.post", - "pk": 3186, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.822Z", - "title": "Very sweet dance", - "body": "
", - "author": "Ashley1023", - "publication_date": "2020-07-21T13:03:03Z", - "url": "https://www.reddit.com/r/aww/comments/hv5ewq/very_sweet_dance/", - "read": false, - "rule": 81, - "remote_identifier": "hv5ewq" - } -}, -{ - "model": "core.post", - "pk": 3187, - "fields": { - "created": "2020-07-21T20:14:50.776Z", - "modified": "2020-07-21T20:14:50.825Z", - "title": "My local pet-store has a cat named Vegemite \u2764\ufe0f", - "body": "
\"My
", - "author": "galinhad", - "publication_date": "2020-07-21T12:06:17Z", - "url": "https://www.reddit.com/r/aww/comments/hv4s5z/my_local_petstore_has_a_cat_named_vegemite/", - "read": false, - "rule": 81, - "remote_identifier": "hv4s5z" - } -}, -{ - "model": "core.post", - "pk": 3188, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-21T20:15:01.459Z", - "title": "A teacher like that makes a huge difference", - "body": "
", - "author": "Unicornglitteryblood", - "publication_date": "2020-07-21T18:29:57Z", - "url": "https://www.reddit.com/r/aww/comments/hvajo9/a_teacher_like_that_makes_a_huge_difference/", - "read": true, - "rule": 81, - "remote_identifier": "hvajo9" - } -}, -{ - "model": "core.post", - "pk": 3189, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-22T19:55:49.930Z", - "title": "Kitten Encounters Bubbly Water", - "body": "
\"Kitten
", - "author": "DragonOBunny", - "publication_date": "2020-07-21T15:28:05Z", - "url": "https://www.reddit.com/r/aww/comments/hv7bis/kitten_encounters_bubbly_water/", - "read": true, - "rule": 81, - "remote_identifier": "hv7bis" - } -}, -{ - "model": "core.post", - "pk": 3190, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-21T20:14:50.833Z", - "title": "Are These My Chickens Now?", - "body": "", - "author": "jasontaken", - "publication_date": "2020-07-21T09:55:36Z", - "url": "https://www.reddit.com/r/aww/comments/hv3de1/are_these_my_chickens_now/", - "read": false, - "rule": 81, - "remote_identifier": "hv3de1" - } -}, -{ - "model": "core.post", - "pk": 3191, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-25T20:08:20.518Z", - "title": "Our St Bernard 6 months apart", - "body": "
\"Our
", - "author": "ryan3105", - "publication_date": "2020-07-21T18:00:04Z", - "url": "https://www.reddit.com/r/aww/comments/hv9yea/our_st_bernard_6_months_apart/", - "read": true, - "rule": 81, - "remote_identifier": "hv9yea" - } -}, -{ - "model": "core.post", - "pk": 3192, - "fields": { - "created": "2020-07-21T20:14:50.777Z", - "modified": "2020-07-21T20:14:50.837Z", - "title": "Father and child in sync", - "body": "
", - "author": "Araragi_Monogatari", - "publication_date": "2020-07-21T08:29:18Z", - "url": "https://www.reddit.com/r/aww/comments/hv2enj/father_and_child_in_sync/", - "read": false, - "rule": 81, - "remote_identifier": "hv2enj" - } -}, -{ - "model": "core.post", - "pk": 3193, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.840Z", - "title": "A meme is born", - "body": "
\"A
", - "author": "Unicornglitteryblood", - "publication_date": "2020-07-21T18:55:04Z", - "url": "https://www.reddit.com/r/aww/comments/hvb1vh/a_meme_is_born/", - "read": false, - "rule": 81, - "remote_identifier": "hvb1vh" - } -}, -{ - "model": "core.post", - "pk": 3194, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.842Z", - "title": "She bites, then she sleeps, then bites again, then sleeps again. \ud83d\ude02", - "body": "
", - "author": "earlymauvs", - "publication_date": "2020-07-21T11:34:19Z", - "url": "https://www.reddit.com/r/aww/comments/hv4fat/she_bites_then_she_sleeps_then_bites_again_then/", - "read": false, - "rule": 81, - "remote_identifier": "hv4fat" - } -}, -{ - "model": "core.post", - "pk": 3195, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.844Z", - "title": "Nothing calmer that 2 ginger cats rubbing heads and showing their love in morning", - "body": "
\"Nothing
", - "author": "Apotheosis33", - "publication_date": "2020-07-21T08:39:24Z", - "url": "https://www.reddit.com/r/aww/comments/hv2j2g/nothing_calmer_that_2_ginger_cats_rubbing_heads/", - "read": false, - "rule": 81, - "remote_identifier": "hv2j2g" - } -}, -{ - "model": "core.post", - "pk": 3196, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.851Z", - "title": "Ring Tailed Possum", - "body": "", - "author": "Wayward-Delver", - "publication_date": "2020-07-21T11:23:51Z", - "url": "https://www.reddit.com/r/aww/comments/hv4b9e/ring_tailed_possum/", - "read": false, - "rule": 81, - "remote_identifier": "hv4b9e" - } -}, -{ - "model": "core.post", - "pk": 3197, - "fields": { - "created": "2020-07-21T20:14:50.778Z", - "modified": "2020-07-21T20:14:50.854Z", - "title": "Baby scooby in sad mood....", - "body": "
\"Baby
", - "author": "deepanshuahiroo7", - "publication_date": "2020-07-21T15:12:23Z", - "url": "https://www.reddit.com/r/aww/comments/hv73ft/baby_scooby_in_sad_mood/", - "read": false, - "rule": 81, - "remote_identifier": "hv73ft" - } -}, -{ - "model": "core.post", - "pk": 3198, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:14:50.856Z", - "title": "New friends!", - "body": "
\"New
", - "author": "HelentotheKeller", - "publication_date": "2020-07-21T13:10:48Z", - "url": "https://www.reddit.com/r/aww/comments/hv5i6i/new_friends/", - "read": false, - "rule": 81, - "remote_identifier": "hv5i6i" - } -}, -{ - "model": "core.post", - "pk": 3199, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:14:50.858Z", - "title": "When you haven't chewed anything for 1 second", - "body": "
\"When
", - "author": "Tanay4", - "publication_date": "2020-07-21T10:26:53Z", - "url": "https://www.reddit.com/r/aww/comments/hv3pl0/when_you_havent_chewed_anything_for_1_second/", - "read": false, - "rule": 81, - "remote_identifier": "hv3pl0" - } -}, -{ - "model": "core.post", - "pk": 3200, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:17:01.490Z", - "title": "Mango Derp", - "body": "
\"Mango
", - "author": "sheetglass", - "publication_date": "2020-07-21T13:27:26Z", - "url": "https://www.reddit.com/r/aww/comments/hv5p8s/mango_derp/", - "read": true, - "rule": 81, - "remote_identifier": "hv5p8s" - } -}, -{ - "model": "core.post", - "pk": 3201, - "fields": { - "created": "2020-07-21T20:14:50.779Z", - "modified": "2020-07-21T20:14:50.863Z", - "title": "My guy turns 20 next month", - "body": "
\"My
", - "author": "alozsoc", - "publication_date": "2020-07-21T06:34:26Z", - "url": "https://www.reddit.com/r/aww/comments/hv0xp1/my_guy_turns_20_next_month/", - "read": false, - "rule": 81, - "remote_identifier": "hv0xp1" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "add_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "change_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "delete_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view log entry", - "content_type": [ - "admin", - "logentry" - ], - "codename": "view_logentry" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "add_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "change_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "delete_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view permission", - "content_type": [ - "auth", - "permission" - ], - "codename": "view_permission" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add group", - "content_type": [ - "auth", - "group" - ], - "codename": "add_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change group", - "content_type": [ - "auth", - "group" - ], - "codename": "change_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete group", - "content_type": [ - "auth", - "group" - ], - "codename": "delete_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view group", - "content_type": [ - "auth", - "group" - ], - "codename": "view_group" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "add_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "change_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "delete_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view content type", - "content_type": [ - "contenttypes", - "contenttype" - ], - "codename": "view_contenttype" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add session", - "content_type": [ - "sessions", - "session" - ], - "codename": "add_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change session", - "content_type": [ - "sessions", - "session" - ], - "codename": "change_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete session", - "content_type": [ - "sessions", - "session" - ], - "codename": "delete_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view session", - "content_type": [ - "sessions", - "session" - ], - "codename": "view_session" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "add_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "change_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "delete_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view crontab", - "content_type": [ - "django_celery_beat", - "crontabschedule" - ], - "codename": "view_crontabschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "add_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "change_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "delete_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view interval", - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "codename": "view_intervalschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "add_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "change_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "delete_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view periodic task", - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "codename": "view_periodictask" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "add_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "change_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "delete_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view periodic tasks", - "content_type": [ - "django_celery_beat", - "periodictasks" - ], - "codename": "view_periodictasks" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "add_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "change_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "delete_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view solar event", - "content_type": [ - "django_celery_beat", - "solarschedule" - ], - "codename": "view_solarschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "add_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "change_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "delete_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view clocked", - "content_type": [ - "django_celery_beat", - "clockedschedule" - ], - "codename": "view_clockedschedule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "add_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "change_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "delete_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view registration profile", - "content_type": [ - "registration", - "registrationprofile" - ], - "codename": "view_registrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "add_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "change_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "delete_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view supervised registration profile", - "content_type": [ - "registration", - "supervisedregistrationprofile" - ], - "codename": "view_supervisedregistrationprofile" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "add_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "change_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "delete_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view access attempt", - "content_type": [ - "axes", - "accessattempt" - ], - "codename": "view_accessattempt" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "add_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "change_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "delete_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view access log", - "content_type": [ - "axes", - "accesslog" - ], - "codename": "view_accesslog" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add user", - "content_type": [ - "accounts", - "user" - ], - "codename": "add_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change user", - "content_type": [ - "accounts", - "user" - ], - "codename": "change_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete user", - "content_type": [ - "accounts", - "user" - ], - "codename": "delete_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view user", - "content_type": [ - "accounts", - "user" - ], - "codename": "view_user" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add post", - "content_type": [ - "core", - "post" - ], - "codename": "add_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change post", - "content_type": [ - "core", - "post" - ], - "codename": "change_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete post", - "content_type": [ - "core", - "post" - ], - "codename": "delete_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view post", - "content_type": [ - "core", - "post" - ], - "codename": "view_post" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add Category", - "content_type": [ - "core", - "category" - ], - "codename": "add_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change Category", - "content_type": [ - "core", - "category" - ], - "codename": "change_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete Category", - "content_type": [ - "core", - "category" - ], - "codename": "delete_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view Category", - "content_type": [ - "core", - "category" - ], - "codename": "view_category" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can add collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "add_collectionrule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can change collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "change_collectionrule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can delete collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "delete_collectionrule" - } -}, -{ - "model": "auth.permission", - "fields": { - "name": "Can view collection rule", - "content_type": [ - "collection", - "collectionrule" - ], - "codename": "view_collectionrule" - } -}, -{ - "model": "accounts.user", - "fields": { - "password": "pbkdf2_sha256$180000$U9a2CS9X0b8Y$T6bD/VoUOFoGNIp16aFlOL0N7q0e6A3I97ypm/AhsGo=", - "last_login": "2020-07-21T20:14:35.966Z", - "is_superuser": true, - "first_name": "", - "last_name": "", - "is_staff": true, - "is_active": true, - "date_joined": "2019-07-18T18:52:36.080Z", - "email": "sonny@bakker.nl", - "task": 10, - "reddit_refresh_token": null, - "reddit_access_token": null, - "groups": [], - "user_permissions": [] - } -}, -{ - "model": "core.category", - "pk": 8, - "fields": { - "created": "2019-11-17T19:37:24.671Z", - "modified": "2019-11-18T19:59:55.010Z", - "name": "World news", - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "core.category", - "pk": 9, - "fields": { - "created": "2019-11-17T19:37:26.161Z", - "modified": "2020-05-30T13:36:10.509Z", - "name": "Tech", - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 3, - "fields": { - "created": "2019-07-14T13:08:10.374Z", - "modified": "2020-07-14T11:45:30.680Z", - "name": "Hackers News", - "type": "feed", - "url": "https://news.ycombinator.com/rss", - "website_url": "https://news.ycombinator.com/", - "favicon": "https://news.ycombinator.com/favicon.ico", - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:30.477Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 4, - "fields": { - "created": "2019-07-20T11:24:32.745Z", - "modified": "2020-07-14T11:45:29.357Z", - "name": "BBC", - "type": "feed", - "url": "http://feeds.bbci.co.uk/news/world/rss.xml", - "website_url": "https://www.bbc.co.uk/news/", - "favicon": "https://m.files.bbci.co.uk/modules/bbc-morph-news-waf-page-meta/2.5.2/apple-touch-icon-57x57-precomposed.png", - "timezone": "UTC", - "category": 8, - "last_suceeded": "2020-07-14T11:45:28.863Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 5, - "fields": { - "created": "2019-07-20T11:24:50.411Z", - "modified": "2020-07-14T11:45:30.063Z", - "name": "Ars Technica", - "type": "feed", - "url": "http://feeds.arstechnica.com/arstechnica/index?fmt=xml", - "website_url": "https://arstechnica.com", - "favicon": "https://cdn.arstechnica.net/favicon.ico", - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:29.810Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 6, - "fields": { - "created": "2019-07-20T11:25:02.089Z", - "modified": "2020-07-14T11:45:30.473Z", - "name": "The Guardian", - "type": "feed", - "url": "https://www.theguardian.com/world/rss", - "website_url": "https://www.theguardian.com/world", - "favicon": "https://assets.guim.co.uk/images/favicons/873381bf11d58e20f551905d51575117/72x72.png", - "timezone": "UTC", - "category": 8, - "last_suceeded": "2020-07-14T11:45:30.181Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 7, - "fields": { - "created": "2019-07-20T11:25:30.121Z", - "modified": "2020-07-14T11:45:29.807Z", - "name": "Tweakers", - "type": "feed", - "url": "http://feeds.feedburner.com/tweakers/mixed?fmt=xml", - "website_url": "https://tweakers.net/", - "favicon": null, - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:29.525Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 8, - "fields": { - "created": "2019-07-20T11:25:46.256Z", - "modified": "2020-07-14T11:45:30.179Z", - "name": "The Verge", - "type": "feed", - "url": "https://www.theverge.com/rss/index.xml", - "website_url": "https://www.theverge.com/", - "favicon": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395367/favicon-16x16.0.png", - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-14T11:45:30.066Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 9, - "fields": { - "created": "2019-11-24T15:28:41.399Z", - "modified": "2020-07-14T11:45:29.522Z", - "name": "NOS", - "type": "feed", - "url": "http://feeds.nos.nl/nosnieuwsalgemeen", - "website_url": null, - "favicon": null, - "timezone": "Europe/Amsterdam", - "category": 8, - "last_suceeded": "2020-07-14T11:45:29.362Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 80, - "fields": { - "created": "2020-07-08T19:30:10.638Z", - "modified": "2020-07-21T20:14:50.609Z", - "name": "Linux subreddit", - "type": "subreddit", - "url": "https://oauth.reddit.com/r/linux/hot", - "website_url": null, - "favicon": null, - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-21T20:14:50.492Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 81, - "fields": { - "created": "2020-07-08T19:30:33.590Z", - "modified": "2020-07-21T20:14:50.865Z", - "name": "AWW subreddit", - "type": "subreddit", - "url": "https://oauth.reddit.com/r/aww/hot", - "website_url": null, - "favicon": null, - "timezone": "UTC", - "category": 8, - "last_suceeded": "2020-07-21T20:14:50.768Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "collection.collectionrule", - "pk": 82, - "fields": { - "created": "2020-07-20T19:29:37.675Z", - "modified": "2020-07-21T20:14:50.489Z", - "name": "Star citizen subreddit", - "type": "subreddit", - "url": "https://oauth.reddit.com/r/starcitizen/hot.json", - "website_url": null, - "favicon": null, - "timezone": "UTC", - "category": 9, - "last_suceeded": "2020-07-21T20:14:50.355Z", - "succeeded": true, - "error": null, - "enabled": true, - "user": [ - "sonny@bakker.nl" - ] - } -}, -{ - "model": "admin.logentry", - "pk": 1, - "fields": { - "action_time": "2020-05-24T18:38:44.624Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "intervalschedule" - ], - "object_id": "5", - "object_repr": "every 4 hours", - "action_flag": 1, - "change_message": "[{\"added\": {}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 2, - "fields": { - "action_time": "2020-05-24T18:38:46.689Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "10", - "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Interval Schedule\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 3, - "fields": { - "action_time": "2020-05-24T18:39:09.203Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "26", - "object_repr": "sonnyba871@gmail.com-collection-task: every hour", - "action_flag": 3, - "change_message": "" - } -}, -{ - "model": "admin.logentry", - "pk": 4, - "fields": { - "action_time": "2020-05-24T19:46:50.248Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "10", - "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Positional Arguments\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 5, - "fields": { - "action_time": "2020-07-07T19:37:57.086Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Reddit refresh token\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 6, - "fields": { - "action_time": "2020-07-07T19:39:46.160Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "10", - "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Task (registered)\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 7, - "fields": { - "action_time": "2020-07-08T19:29:27.025Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "django_celery_beat", - "periodictask" - ], - "object_id": "11", - "object_repr": "Reddit collection task: every 4 hours", - "action_flag": 1, - "change_message": "[{\"added\": {}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 8, - "fields": { - "action_time": "2020-07-14T11:46:50.039Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 9, - "fields": { - "action_time": "2020-07-18T19:08:33.997Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "collection", - "collectionrule" - ], - "object_id": "81", - "object_repr": "AWW subreddit", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 10, - "fields": { - "action_time": "2020-07-18T19:08:44.063Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "collection", - "collectionrule" - ], - "object_id": "80", - "object_repr": "Linux subreddit", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 11, - "fields": { - "action_time": "2020-07-18T19:17:25.213Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2336", - "object_repr": "Post-2336", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 12, - "fields": { - "action_time": "2020-07-18T19:17:40.596Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2336", - "object_repr": "Post-2336", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 13, - "fields": { - "action_time": "2020-07-19T10:55:55.807Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2764", - "object_repr": "Post-2764", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 14, - "fields": { - "action_time": "2020-07-19T10:57:40.643Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2764", - "object_repr": "Post-2764", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 15, - "fields": { - "action_time": "2020-07-19T10:58:05.823Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "core", - "post" - ], - "object_id": "2764", - "object_repr": "Post-2764", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 16, - "fields": { - "action_time": "2020-07-26T09:51:52.478Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 17, - "fields": { - "action_time": "2020-07-26T09:52:04.691Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"password\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 18, - "fields": { - "action_time": "2020-07-26T09:52:12.392Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" - } -}, -{ - "model": "admin.logentry", - "pk": 19, - "fields": { - "action_time": "2020-07-26T09:56:15.949Z", - "user": [ - "sonny@bakker.nl" - ], - "content_type": [ - "accounts", - "user" - ], - "object_id": "1", - "object_repr": "sonny@bakker.nl", - "action_flag": 2, - "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" - } -} -] +[ +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "admin", + "model": "logentry" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", + "model": "permission" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", + "model": "group" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "contenttypes", + "model": "contenttype" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "sessions", + "model": "session" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "crontabschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "intervalschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "periodictask" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "periodictasks" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "solarschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "django_celery_beat", + "model": "clockedschedule" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "registration", + "model": "registrationprofile" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "registration", + "model": "supervisedregistrationprofile" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "axes", + "model": "accessattempt" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "axes", + "model": "accesslog" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "accounts", + "model": "user" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "core", + "model": "post" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "core", + "model": "category" + } +}, +{ + "model": "contenttypes.contenttype", + "fields": { + "app_label": "collection", + "model": "collectionrule" + } +}, +{ + "model": "sessions.session", + "pk": "3sumq22krk8tsvexcs4b8czu82yhvuer", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-05-16T18:29:04.049Z" + } +}, +{ + "model": "sessions.session", + "pk": "8ix6bdwf2ywk0eir1hb062dhfh9xit85", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-07-21T19:36:54.530Z" + } +}, +{ + "model": "sessions.session", + "pk": "d4wophwpjm8z96doe8iddvhdv9yfafyx", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-06-07T19:45:49.727Z" + } +}, +{ + "model": "sessions.session", + "pk": "g23ziz66li5zx8nd8cewb3vevdxhjkm0", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-06-30T06:55:50.747Z" + } +}, +{ + "model": "sessions.session", + "pk": "jwn66dptmdkm6hom2ns3j288aaxqtyjd", + "fields": { + "session_data": "OWZkZTQyZDQ2NzNkYzdkOTBhM2ZlOWU3MDhhNDkyMWQ0MDdmZTc5ODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiJhZTMwMWFlMzI5OGFlOThkNjY1MTY1NDIxM2EyMmM0NDA0Y2FkZTc3In0=", + "expire_date": "2020-06-07T18:38:19.116Z" + } +}, +{ + "model": "sessions.session", + "pk": "wjz6kwg5e5ciemre0l0wwyrcwcj2gyg6", + "fields": { + "session_data": "MWU5ODBjY2QyOTFhMmRiY2QyYjQwZjQ3MmMwYmExYjBlYTkxNTcwODp7Il9hdXRoX3VzZXJfaWQiOiIxIiwiX2F1dGhfdXNlcl9iYWNrZW5kIjoiZGphbmdvLmNvbnRyaWIuYXV0aC5iYWNrZW5kcy5Nb2RlbEJhY2tlbmQiLCJfYXV0aF91c2VyX2hhc2giOiI0YWZkYTkxNzU5ZDBhZDZmMjg1ZTQyOGY0OTUxN2M5MTJhMmM5NWIyIn0=", + "expire_date": "2020-08-09T09:52:04.705Z" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 1, + "fields": { + "every": 5, + "period": "minutes" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 2, + "fields": { + "every": 15, + "period": "minutes" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 3, + "fields": { + "every": 30, + "period": "minutes" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 4, + "fields": { + "every": 1, + "period": "hours" + } +}, +{ + "model": "django_celery_beat.intervalschedule", + "pk": 5, + "fields": { + "every": 4, + "period": "hours" + } +}, +{ + "model": "django_celery_beat.crontabschedule", + "pk": 1, + "fields": { + "minute": "0", + "hour": "4", + "day_of_week": "*", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "UTC" + } +}, +{ + "model": "django_celery_beat.periodictasks", + "pk": 1, + "fields": { + "last_update": "2020-07-26T09:47:48.298Z" + } +}, +{ + "model": "django_celery_beat.periodictask", + "pk": 1, + "fields": { + "name": "celery.backend_cleanup", + "task": "celery.backend_cleanup", + "interval": null, + "crontab": 1, + "solar": null, + "clocked": null, + "args": "[]", + "kwargs": "{}", + "queue": null, + "exchange": null, + "routing_key": null, + "headers": "{}", + "priority": null, + "expires": null, + "expire_seconds": 43200, + "one_off": false, + "start_time": null, + "enabled": true, + "last_run_at": "2020-07-26T09:47:48.322Z", + "total_run_count": 17, + "date_changed": "2020-07-26T09:47:50.362Z", + "description": "" + } +}, +{ + "model": "django_celery_beat.periodictask", + "pk": 10, + "fields": { + "name": "sonny@bakker.nl-collection-task", + "task": "FeedTask", + "interval": 5, + "crontab": null, + "solar": null, + "clocked": null, + "args": "[1]", + "kwargs": "{}", + "queue": null, + "exchange": null, + "routing_key": null, + "headers": "{}", + "priority": null, + "expires": null, + "expire_seconds": null, + "one_off": false, + "start_time": null, + "enabled": false, + "last_run_at": "2020-07-14T11:45:26.209Z", + "total_run_count": 307, + "date_changed": "2020-07-14T11:45:41.282Z", + "description": "" + } +}, +{ + "model": "django_celery_beat.periodictask", + "pk": 11, + "fields": { + "name": "Reddit collection task", + "task": "RedditTask", + "interval": 5, + "crontab": null, + "solar": null, + "clocked": null, + "args": "[]", + "kwargs": "{}", + "queue": null, + "exchange": null, + "routing_key": null, + "headers": "{}", + "priority": null, + "expires": null, + "expire_seconds": null, + "one_off": false, + "start_time": null, + "enabled": false, + "last_run_at": null, + "total_run_count": 4, + "date_changed": "2020-07-14T11:45:41.316Z", + "description": "" + } +}, +{ + "model": "core.post", + "pk": 3061, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:14:50.423Z", + "title": "Star Citizen: Question and Answer Thread", + "body": "

Welcome to the Star Citizen question and answer thread. Feel free to ask any questions you have related to SC here!

\n\n\n\n

Useful Links and Resources:

\n\n

Star Citizen Wiki - The biggest and best wiki resource dedicated to Star Citizen

\n\n

Star Citizen FAQ - Chances the answer you need is here.

\n\n

Discord Help Channel - Often times community members will be here to help you with issues.

\n\n

Referral Code Randomizer - Use this when creating a new account to get 5000 extra UEC.

\n\n

Download Star Citizen - Get the latest version of Star Citizen here

\n\n

Current Game Features - Click here to see what you can currently do in Star Citizen.

\n\n

Development Roadmap - The current development status of up and coming Star Citizen features.

\n\n

Pledge FAQ - Official FAQ regarding spending money on the game.

\n
", + "author": "UEE_Central_Computer", + "publication_date": "2020-07-20T14:00:10Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huk04t/star_citizen_question_and_answer_thread/", + "read": false, + "rule": 82, + "remote_identifier": "huk04t" + } +}, +{ + "model": "core.post", + "pk": 3062, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:33:37.019Z", + "title": "Peace and Quiet", + "body": "
\"Peace
", + "author": "SourMemeNZ", + "publication_date": "2020-07-20T14:09:49Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huk4ib/peace_and_quiet/", + "read": true, + "rule": 82, + "remote_identifier": "huk4ib" + } +}, +{ + "model": "core.post", + "pk": 3063, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:14:50.463Z", + "title": "Y'all are probably sick of em by now but here's my LEGO Mercury Star Runner (MSR).", + "body": "
\"Y'all
", + "author": "osamadabinman", + "publication_date": "2020-07-20T19:53:23Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hupzqa/yall_are_probably_sick_of_em_by_now_but_heres_my/", + "read": true, + "rule": 82, + "remote_identifier": "hupzqa" + } +}, +{ + "model": "core.post", + "pk": 3064, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:12.253Z", + "title": "Damned Space Invaders and their pixel weapons!", + "body": "
\"Damned
", + "author": "Akaradrin", + "publication_date": "2020-07-20T14:26:18Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hukckf/damned_space_invaders_and_their_pixel_weapons/", + "read": true, + "rule": 82, + "remote_identifier": "hukckf" + } +}, +{ + "model": "core.post", + "pk": 3065, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.578Z", + "title": "The sky is no longer the limit", + "body": "
\"The
", + "author": "CyberTill", + "publication_date": "2020-07-20T14:11:31Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huk5b8/the_sky_is_no_longer_the_limit/", + "read": false, + "rule": 82, + "remote_identifier": "huk5b8" + } +}, +{ + "model": "core.post", + "pk": 3066, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:23.282Z", + "title": "Terrapin Hover Mode Gameplay [Full Video in Comments]", + "body": "
", + "author": "Didactic_Tomato", + "publication_date": "2020-07-20T11:01:13Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hui1gv/terrapin_hover_mode_gameplay_full_video_in/", + "read": true, + "rule": 82, + "remote_identifier": "hui1gv" + } +}, +{ + "model": "core.post", + "pk": 3067, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:44.250Z", + "title": "honestly", + "body": "
\"honestly\"
", + "author": "Beatlead", + "publication_date": "2020-07-20T18:24:07Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huo96t/honestly/", + "read": true, + "rule": 82, + "remote_identifier": "huo96t" + } +}, +{ + "model": "core.post", + "pk": 3068, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.584Z", + "title": "As a paranoiac and tired of checking if door was closed, saved to f4 thoses \"security cam\" positions, could be usefull for larger ships :)", + "body": "", + "author": "icwiener__", + "publication_date": "2020-07-20T13:03:33Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hujchz/as_a_paranoiac_and_tired_of_checking_if_door_was/", + "read": false, + "rule": 82, + "remote_identifier": "hujchz" + } +}, +{ + "model": "core.post", + "pk": 3069, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:33:59.158Z", + "title": "Station Manager: \"You're too fat, we won't let you in, go and fall on Lorville. Thank you for your call!\" Me: \"okay :'(\"", + "body": "
\"Station
", + "author": "Shaman_N_One", + "publication_date": "2020-07-20T11:33:38Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huidlu/station_manager_youre_too_fat_we_wont_let_you_in/", + "read": true, + "rule": 82, + "remote_identifier": "huidlu" + } +}, +{ + "model": "core.post", + "pk": 3070, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.588Z", + "title": "[PTU Bug Hunt Request] Packet Loss", + "body": "", + "author": "Rainwalker007", + "publication_date": "2020-07-20T18:38:03Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huoicq/ptu_bug_hunt_request_packet_loss/", + "read": false, + "rule": 82, + "remote_identifier": "huoicq" + } +}, +{ + "model": "core.post", + "pk": 3071, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:17:52.092Z", + "title": "Anyone able to explain these \"trail frames\"?", + "body": "
\"Anyone
", + "author": "Abnormal_Sloth", + "publication_date": "2020-07-20T17:11:32Z", + "url": "https://www.reddit.com/r/starcitizen/comments/humyeq/anyone_able_to_explain_these_trail_frames/", + "read": true, + "rule": 82, + "remote_identifier": "humyeq" + } +}, +{ + "model": "core.post", + "pk": 3072, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.593Z", + "title": "#BringBackBugSmasher - A long forgotten legendary video content", + "body": "", + "author": "MasterBoring", + "publication_date": "2020-07-20T18:05:54Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hunx77/bringbackbugsmasher_a_long_forgotten_legendary/", + "read": false, + "rule": 82, + "remote_identifier": "hunx77" + } +}, +{ + "model": "core.post", + "pk": 3073, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:33:22.601Z", + "title": "Oracle Helmet [in-game screenshot; downsampled to 4k]", + "body": "
\"Oracle
", + "author": "mr-hasgaha", + "publication_date": "2020-07-20T17:39:34Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hung0b/oracle_helmet_ingame_screenshot_downsampled_to_4k/", + "read": true, + "rule": 82, + "remote_identifier": "hung0b" + } +}, +{ + "model": "core.post", + "pk": 3074, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:34:42.578Z", + "title": "Testing 3.10 - Gladius in decoupled mode", + "body": "
", + "author": "DarkConstant", + "publication_date": "2020-07-19T21:26:52Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hu6f1h/testing_310_gladius_in_decoupled_mode/", + "read": true, + "rule": 82, + "remote_identifier": "hu6f1h" + } +}, +{ + "model": "core.post", + "pk": 3075, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:34:29.424Z", + "title": "Day 3, I can't stop taking pictures with my Carrack. Send help", + "body": "
\"Day
", + "author": "CyberTill", + "publication_date": "2020-07-20T01:58:15Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huazyy/day_3_i_cant_stop_taking_pictures_with_my_carrack/", + "read": true, + "rule": 82, + "remote_identifier": "huazyy" + } +}, +{ + "model": "core.post", + "pk": 3076, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.602Z", + "title": "I used to enjoy flying between the buildings of new babbage, I mean before the NFZ \"improvement\"", + "body": "
\"I
", + "author": "shoeii", + "publication_date": "2020-07-20T16:40:26Z", + "url": "https://www.reddit.com/r/starcitizen/comments/humet2/i_used_to_enjoy_flying_between_the_buildings_of/", + "read": false, + "rule": 82, + "remote_identifier": "humet2" + } +}, +{ + "model": "core.post", + "pk": 3077, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-21T20:18:04.237Z", + "title": "Thank you CIG for updated heightmaps and render distances", + "body": "
\"Thank
", + "author": "u7f76", + "publication_date": "2020-07-19T23:38:22Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hu8pwf/thank_you_cig_for_updated_heightmaps_and_render/", + "read": true, + "rule": 82, + "remote_identifier": "hu8pwf" + } +}, +{ + "model": "core.post", + "pk": 3078, + "fields": { + "created": "2020-07-20T19:32:35.562Z", + "modified": "2020-07-20T19:32:35.607Z", + "title": "This Week in Star Citizen | July 20th 2020", + "body": "", + "author": "ivtiprogamer", + "publication_date": "2020-07-20T19:50:29Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hupxnt/this_week_in_star_citizen_july_20th_2020/", + "read": false, + "rule": 82, + "remote_identifier": "hupxnt" + } +}, +{ + "model": "core.post", + "pk": 3079, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:34:36.068Z", + "title": "Bravo CIG lighting team! Noticeable improvements to all around environment lighting in 3.10", + "body": "
\"Bravo
", + "author": "u7f76", + "publication_date": "2020-07-20T00:02:23Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hu94o0/bravo_cig_lighting_team_noticeable_improvements/", + "read": true, + "rule": 82, + "remote_identifier": "hu94o0" + } +}, +{ + "model": "core.post", + "pk": 3080, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.613Z", + "title": "Thick", + "body": "
\"Thick\"
", + "author": "burgerbagel", + "publication_date": "2020-07-20T16:24:38Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hum50f/thick/", + "read": false, + "rule": 82, + "remote_identifier": "hum50f" + } +}, +{ + "model": "core.post", + "pk": 3081, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:34:19.763Z", + "title": "Soon\u2122", + "body": "
\"Soon\u2122\"
", + "author": "Mistralette", + "publication_date": "2020-07-20T05:54:09Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hueg01/soon/", + "read": true, + "rule": 82, + "remote_identifier": "hueg01" + } +}, +{ + "model": "core.post", + "pk": 3082, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.618Z", + "title": "On the prowl", + "body": "
\"On
", + "author": "SaraCaterina", + "publication_date": "2020-07-20T16:37:03Z", + "url": "https://www.reddit.com/r/starcitizen/comments/humcmb/on_the_prowl/", + "read": false, + "rule": 82, + "remote_identifier": "humcmb" + } +}, +{ + "model": "core.post", + "pk": 3083, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:34:07.272Z", + "title": "The Hills Have Eyes", + "body": "
\"The
", + "author": "FallenLordik", + "publication_date": "2020-07-20T11:19:19Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hui8ao/the_hills_have_eyes/", + "read": true, + "rule": 82, + "remote_identifier": "hui8ao" + } +}, +{ + "model": "core.post", + "pk": 3084, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.623Z", + "title": "Worried about longer loading screens? Hit ~ and do r_displayinfo 3", + "body": "
\"Worried
", + "author": "kristokn", + "publication_date": "2020-07-20T10:09:53Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huhif1/worried_about_longer_loading_screens_hit_and_do_r/", + "read": false, + "rule": 82, + "remote_identifier": "huhif1" + } +}, +{ + "model": "core.post", + "pk": 3085, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.625Z", + "title": "My contribution to the wallpaper contest... click for the full effect (3440x1440)", + "body": "
\"My
", + "author": "Dougie_Juice", + "publication_date": "2020-07-20T20:02:31Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huq655/my_contribution_to_the_wallpaper_contest_click/", + "read": false, + "rule": 82, + "remote_identifier": "huq655" + } +}, +{ + "model": "core.post", + "pk": 3086, + "fields": { + "created": "2020-07-20T19:32:35.563Z", + "modified": "2020-07-20T19:32:35.627Z", + "title": "Star Citizen: The Onion (Parody Project)", + "body": "", + "author": "BroadOne", + "publication_date": "2020-07-20T19:19:20Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hupbkj/star_citizen_the_onion_parody_project/", + "read": false, + "rule": 82, + "remote_identifier": "hupbkj" + } +}, +{ + "model": "core.post", + "pk": 3087, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.637Z", + "title": "perfect day to sunbathe", + "body": "
", + "author": "Pedrica1", + "publication_date": "2020-07-20T18:08:17Z", + "url": "https://www.reddit.com/r/aww/comments/hunysb/perfect_day_to_sunbathe/", + "read": false, + "rule": 81, + "remote_identifier": "hunysb" + } +}, +{ + "model": "core.post", + "pk": 3088, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.639Z", + "title": "My dogs face when he sees I'm home", + "body": "
", + "author": "NewReddit_WhoDis", + "publication_date": "2020-07-20T16:45:21Z", + "url": "https://www.reddit.com/r/aww/comments/humhxa/my_dogs_face_when_he_sees_im_home/", + "read": false, + "rule": 81, + "remote_identifier": "humhxa" + } +}, +{ + "model": "core.post", + "pk": 3089, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.641Z", + "title": "Cow loves the scritch machine", + "body": "
", + "author": "Der_Ist", + "publication_date": "2020-07-20T17:36:16Z", + "url": "https://www.reddit.com/r/aww/comments/hundvo/cow_loves_the_scritch_machine/", + "read": false, + "rule": 81, + "remote_identifier": "hundvo" + } +}, +{ + "model": "core.post", + "pk": 3090, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.643Z", + "title": "Can I sit next to you ?", + "body": "
", + "author": "wheezy098", + "publication_date": "2020-07-20T17:55:10Z", + "url": "https://www.reddit.com/r/aww/comments/hunq5h/can_i_sit_next_to_you/", + "read": false, + "rule": 81, + "remote_identifier": "hunq5h" + } +}, +{ + "model": "core.post", + "pk": 3091, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.645Z", + "title": "IS THAT A CUSTOMER? flop flop flop flop .... \" Can I uhh... help you sir?\"", + "body": "
", + "author": "MBMV", + "publication_date": "2020-07-20T12:50:40Z", + "url": "https://www.reddit.com/r/aww/comments/huj7g3/is_that_a_customer_flop_flop_flop_flop_can_i_uhh/", + "read": false, + "rule": 81, + "remote_identifier": "huj7g3" + } +}, +{ + "model": "core.post", + "pk": 3092, + "fields": { + "created": "2020-07-20T19:32:35.635Z", + "modified": "2020-07-20T19:32:35.647Z", + "title": "Good Boy turned Disney Princess", + "body": "
", + "author": "Sauwercraud", + "publication_date": "2020-07-20T18:40:05Z", + "url": "https://www.reddit.com/r/aww/comments/huojq0/good_boy_turned_disney_princess/", + "read": false, + "rule": 81, + "remote_identifier": "huojq0" + } +}, +{ + "model": "core.post", + "pk": 3093, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.649Z", + "title": "Kitty loop", + "body": "
", + "author": "Dlatrex", + "publication_date": "2020-07-20T12:54:02Z", + "url": "https://www.reddit.com/r/aww/comments/huj8s6/kitty_loop/", + "read": false, + "rule": 81, + "remote_identifier": "huj8s6" + } +}, +{ + "model": "core.post", + "pk": 3094, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.652Z", + "title": "if i fits i sits", + "body": "
", + "author": "jasontaken", + "publication_date": "2020-07-20T16:38:32Z", + "url": "https://www.reddit.com/r/aww/comments/humdlf/if_i_fits_i_sits/", + "read": false, + "rule": 81, + "remote_identifier": "humdlf" + } +}, +{ + "model": "core.post", + "pk": 3095, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.654Z", + "title": "Isn\u2019t she Adorable !", + "body": "
\"Isn\u2019t
", + "author": "MunchyMac", + "publication_date": "2020-07-20T16:18:05Z", + "url": "https://www.reddit.com/r/aww/comments/hum133/isnt_she_adorable/", + "read": false, + "rule": 81, + "remote_identifier": "hum133" + } +}, +{ + "model": "core.post", + "pk": 3096, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.655Z", + "title": "Thank you mama (\u2283\uff61\u2022\u0301\u203f\u2022\u0300\uff61)\u2283", + "body": "
", + "author": "AnoushkaSingh", + "publication_date": "2020-07-20T13:35:51Z", + "url": "https://www.reddit.com/r/aww/comments/hujpxy/thank_you_mama/", + "read": false, + "rule": 81, + "remote_identifier": "hujpxy" + } +}, +{ + "model": "core.post", + "pk": 3097, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.657Z", + "title": "I WANT TO HUG HIM SO BAD!!!", + "body": "
", + "author": "BATMAN_5777", + "publication_date": "2020-07-20T18:25:20Z", + "url": "https://www.reddit.com/r/aww/comments/huo9z4/i_want_to_hug_him_so_bad/", + "read": false, + "rule": 81, + "remote_identifier": "huo9z4" + } +}, +{ + "model": "core.post", + "pk": 3098, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.659Z", + "title": "Before and after being called a good boy", + "body": "
\"Before
", + "author": "vladgrinch", + "publication_date": "2020-07-20T10:48:40Z", + "url": "https://www.reddit.com/r/aww/comments/huhwu9/before_and_after_being_called_a_good_boy/", + "read": false, + "rule": 81, + "remote_identifier": "huhwu9" + } +}, +{ + "model": "core.post", + "pk": 3099, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.662Z", + "title": "My fianc\u00e9 has wanted a dog his whole life. This is his college graduation present. Welcome home Maple!", + "body": "
\"My
", + "author": "AlexisaurusRex", + "publication_date": "2020-07-20T17:57:25Z", + "url": "https://www.reddit.com/r/aww/comments/hunrie/my_fianc\u00e9_has_wanted_a_dog_his_whole_life_this_is/", + "read": false, + "rule": 81, + "remote_identifier": "hunrie" + } +}, +{ + "model": "core.post", + "pk": 3100, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.664Z", + "title": "Cute burro.", + "body": "
\"Cute
", + "author": "Craftmine101", + "publication_date": "2020-07-20T13:45:32Z", + "url": "https://www.reddit.com/r/aww/comments/huju40/cute_burro/", + "read": false, + "rule": 81, + "remote_identifier": "huju40" + } +}, +{ + "model": "core.post", + "pk": 3101, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.666Z", + "title": "I've never seen anyone dance better than that turtle.", + "body": "
", + "author": "Ashley1023", + "publication_date": "2020-07-20T18:07:30Z", + "url": "https://www.reddit.com/r/aww/comments/hunya8/ive_never_seen_anyone_dance_better_than_that/", + "read": false, + "rule": 81, + "remote_identifier": "hunya8" + } +}, +{ + "model": "core.post", + "pk": 3102, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.669Z", + "title": "Someone\u2019s going to be quite surprised when he realizes all this new stuff isn\u2019t for him!", + "body": "
\"Someone\u2019s
", + "author": "molly590", + "publication_date": "2020-07-20T15:46:21Z", + "url": "https://www.reddit.com/r/aww/comments/hulikg/someones_going_to_be_quite_surprised_when_he/", + "read": false, + "rule": 81, + "remote_identifier": "hulikg" + } +}, +{ + "model": "core.post", + "pk": 3103, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.671Z", + "title": "my aunt asked me to paint her puppy and I think it turned out so cute!!!", + "body": "
\"my
", + "author": "PineappleLightt", + "publication_date": "2020-07-20T16:39:37Z", + "url": "https://www.reddit.com/r/aww/comments/humea0/my_aunt_asked_me_to_paint_her_puppy_and_i_think/", + "read": false, + "rule": 81, + "remote_identifier": "humea0" + } +}, +{ + "model": "core.post", + "pk": 3104, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.673Z", + "title": "Master Assassin", + "body": "
\"Master
", + "author": "LauWalker", + "publication_date": "2020-07-20T18:47:52Z", + "url": "https://www.reddit.com/r/aww/comments/huop8a/master_assassin/", + "read": false, + "rule": 81, + "remote_identifier": "huop8a" + } +}, +{ + "model": "core.post", + "pk": 3105, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.675Z", + "title": "Every time this tank cleaner cleans out the aquarium, this fish swims over to him looking for pets", + "body": "", + "author": "unnaturalorder", + "publication_date": "2020-07-20T05:29:30Z", + "url": "https://www.reddit.com/r/aww/comments/hue3r0/every_time_this_tank_cleaner_cleans_out_the/", + "read": false, + "rule": 81, + "remote_identifier": "hue3r0" + } +}, +{ + "model": "core.post", + "pk": 3106, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.678Z", + "title": "My girlfriend sent me this while I was at work. And here I was thinking the perfect picture of our dog didn't exist", + "body": "", + "author": "Khuma-zi_Eldrama", + "publication_date": "2020-07-20T19:22:48Z", + "url": "https://www.reddit.com/r/aww/comments/hupdz8/my_girlfriend_sent_me_this_while_i_was_at_work/", + "read": false, + "rule": 81, + "remote_identifier": "hupdz8" + } +}, +{ + "model": "core.post", + "pk": 3107, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.680Z", + "title": "My first ever post, everyone meet my new baby girl Kiora! I\u2019m so in love with her\ud83e\udd7a\ud83d\udcab", + "body": "
\"My
", + "author": "Dumpling2463", + "publication_date": "2020-07-20T05:34:29Z", + "url": "https://www.reddit.com/r/aww/comments/hue6dx/my_first_ever_post_everyone_meet_my_new_baby_girl/", + "read": false, + "rule": 81, + "remote_identifier": "hue6dx" + } +}, +{ + "model": "core.post", + "pk": 3108, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.682Z", + "title": "Dog splashing in water", + "body": "", + "author": "TheRikari", + "publication_date": "2020-07-20T15:44:02Z", + "url": "https://www.reddit.com/r/aww/comments/hulh8k/dog_splashing_in_water/", + "read": false, + "rule": 81, + "remote_identifier": "hulh8k" + } +}, +{ + "model": "core.post", + "pk": 3109, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.685Z", + "title": "They say taking breaks is the key to productivity!", + "body": "
", + "author": "Thereaper29", + "publication_date": "2020-07-20T05:43:40Z", + "url": "https://www.reddit.com/r/aww/comments/hueawt/they_say_taking_breaks_is_the_key_to_productivity/", + "read": false, + "rule": 81, + "remote_identifier": "hueawt" + } +}, +{ + "model": "core.post", + "pk": 3110, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.687Z", + "title": "I went away for 3 weeks, and now my cat is in love with my husband", + "body": "
\"I
", + "author": "sillykittyish", + "publication_date": "2020-07-20T03:29:11Z", + "url": "https://www.reddit.com/r/aww/comments/hucd7u/i_went_away_for_3_weeks_and_now_my_cat_is_in_love/", + "read": false, + "rule": 81, + "remote_identifier": "hucd7u" + } +}, +{ + "model": "core.post", + "pk": 3111, + "fields": { + "created": "2020-07-20T19:32:35.636Z", + "modified": "2020-07-20T19:32:35.689Z", + "title": "Can you feel the love", + "body": "
", + "author": "kettySewrdPic", + "publication_date": "2020-07-20T09:13:32Z", + "url": "https://www.reddit.com/r/aww/comments/hugx1k/can_you_feel_the_love/", + "read": false, + "rule": 81, + "remote_identifier": "hugx1k" + } +}, +{ + "model": "core.post", + "pk": 3112, + "fields": { + "created": "2020-07-20T19:32:35.835Z", + "modified": "2020-07-21T20:14:50.522Z", + "title": "Linux Experiences/Rants or Education/Certifications thread - July 20, 2020", + "body": "

Welcome to r/linux rants and experiences! This megathread is also to hear opinions from anyone just starting out with Linux or those that have used Linux (GNU or otherwise) for a long time.

\n\n

Let us know what's annoying you, whats making you happy, or something that you want to get out to r/linux but didn't make the cut into a full post of it's own.

\n\n

For those looking for certifications please use this megathread to ask about how to get certified whether it's for the business world or for your own satisfaction. Be sure to check out r/linuxadmin for more discussion in the SysAdmin world!

\n\n

Please keep questions in r/linuxquestions, r/linux4noobs, or the Wednesday automod thread.

\n
", + "author": "AutoModerator", + "publication_date": "2020-07-20T06:12:00Z", + "url": "https://www.reddit.com/r/linux/comments/hueoo0/linux_experiencesrants_or_educationcertifications/", + "read": false, + "rule": 80, + "remote_identifier": "hueoo0" + } +}, +{ + "model": "core.post", + "pk": 3113, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:19:49.339Z", + "title": "Unix Family Tree", + "body": "
\"Unix
", + "author": "bauripalash", + "publication_date": "2020-07-20T10:32:15Z", + "url": "https://www.reddit.com/r/linux/comments/huhqrh/unix_family_tree/", + "read": true, + "rule": 80, + "remote_identifier": "huhqrh" + } +}, +{ + "model": "core.post", + "pk": 3114, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:14:50.554Z", + "title": "NVIDIA open sourced part of NVAPI SDK to aid 'Windows emulation environments'", + "body": "", + "author": "ignapk", + "publication_date": "2020-07-20T13:17:19Z", + "url": "https://www.reddit.com/r/linux/comments/huji8c/nvidia_open_sourced_part_of_nvapi_sdk_to_aid/", + "read": false, + "rule": 80, + "remote_identifier": "huji8c" + } +}, +{ + "model": "core.post", + "pk": 3115, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:14:50.551Z", + "title": "Jellyfin 10.6 released", + "body": "", + "author": "resoluti0n_", + "publication_date": "2020-07-20T16:40:05Z", + "url": "https://www.reddit.com/r/linux/comments/humekr/jellyfin_106_released/", + "read": false, + "rule": 80, + "remote_identifier": "humekr" + } +}, +{ + "model": "core.post", + "pk": 3116, + "fields": { + "created": "2020-07-20T19:32:35.836Z", + "modified": "2020-07-21T20:14:50.583Z", + "title": "[German] Article in major german newspaper about trying Linux and WSL. Literal: \"Why it's beneficial to try Linux now\"", + "body": "", + "author": "noname7890", + "publication_date": "2020-07-19T15:19:27Z", + "url": "https://www.reddit.com/r/linux/comments/hu0d5v/german_article_in_major_german_newspaper_about/", + "read": false, + "rule": 80, + "remote_identifier": "hu0d5v" + } +}, +{ + "model": "core.post", + "pk": 3117, + "fields": { + "created": "2020-07-20T19:32:35.837Z", + "modified": "2020-07-21T20:14:50.574Z", + "title": "Brian Kernighan: UNIX, C, AWK, AMPL, and Go Programming | AI Podcast #109 with Lex Fridman", + "body": "", + "author": "tinyatom", + "publication_date": "2020-07-20T08:48:35Z", + "url": "https://www.reddit.com/r/linux/comments/hugn0w/brian_kernighan_unix_c_awk_ampl_and_go/", + "read": false, + "rule": 80, + "remote_identifier": "hugn0w" + } +}, +{ + "model": "core.post", + "pk": 3118, + "fields": { + "created": "2020-07-20T19:32:35.837Z", + "modified": "2020-07-21T20:14:50.578Z", + "title": "Explaining Computers Host Christopher Barnatt Has Switched To Linux", + "body": "", + "author": "sysrpl", + "publication_date": "2020-07-20T13:00:02Z", + "url": "https://www.reddit.com/r/linux/comments/hujb12/explaining_computers_host_christopher_barnatt_has/", + "read": false, + "rule": 80, + "remote_identifier": "hujb12" + } +}, +{ + "model": "core.post", + "pk": 3119, + "fields": { + "created": "2020-07-20T19:32:35.837Z", + "modified": "2020-07-21T20:14:50.529Z", + "title": "Ireland donates contact tracing app to the Linux foundation.", + "body": "", + "author": "mathiasryan", + "publication_date": "2020-07-20T21:31:43Z", + "url": "https://www.reddit.com/r/linux/comments/hury4e/ireland_donates_contact_tracing_app_to_the_linux/", + "read": false, + "rule": 80, + "remote_identifier": "hury4e" + } +}, +{ + "model": "core.post", + "pk": 3120, + "fields": { + "created": "2020-07-20T19:32:35.842Z", + "modified": "2020-07-21T20:14:50.588Z", + "title": "I implemented a simple terminal-based password manager", + "body": "

I created a simple, secure, and free password manager written in C: SaltPass. I haven't contributed open source code before, but I think this might be useful to a few people. Especially as an alternative to paid solutions such as LastPass and the likes. Any suggestions/edits/code improvements would be greatly appreciated!

\n
", + "author": "zaid-gg", + "publication_date": "2020-07-20T07:43:03Z", + "url": "https://www.reddit.com/r/linux/comments/hufula/i_implemented_a_simple_terminalbased_password/", + "read": false, + "rule": 80, + "remote_identifier": "hufula" + } +}, +{ + "model": "core.post", + "pk": 3121, + "fields": { + "created": "2020-07-20T19:32:35.843Z", + "modified": "2020-07-21T20:14:50.593Z", + "title": "Performance analysis of multi services on container Docker, LXC, and LXD - Bulletin of Electrical Engineering and Informatics, Adinda Riztia Putri, Rendy Munadi, Ridha Muldina Negara Adaptive Network\u2026", + "body": "", + "author": "bmullan", + "publication_date": "2020-07-20T11:35:59Z", + "url": "https://www.reddit.com/r/linux/comments/huieio/performance_analysis_of_multi_services_on/", + "read": false, + "rule": 80, + "remote_identifier": "huieio" + } +}, +{ + "model": "core.post", + "pk": 3122, + "fields": { + "created": "2020-07-20T19:32:35.844Z", + "modified": "2020-07-21T20:14:50.602Z", + "title": "Create an Internal PKI using OpenSSL and NitroKey HSM", + "body": "", + "author": "PixelPaulaus", + "publication_date": "2020-07-20T06:18:41Z", + "url": "https://www.reddit.com/r/linux/comments/huerpn/create_an_internal_pki_using_openssl_and_nitrokey/", + "read": false, + "rule": 80, + "remote_identifier": "huerpn" + } +}, +{ + "model": "core.post", + "pk": 3123, + "fields": { + "created": "2020-07-20T19:32:35.844Z", + "modified": "2020-07-20T19:32:35.883Z", + "title": "vopono - run applications via VPNs with temporary network namespaces", + "body": "", + "author": "nivenkos", + "publication_date": "2020-07-19T20:02:57Z", + "url": "https://www.reddit.com/r/linux/comments/hu4vge/vopono_run_applications_via_vpns_with_temporary/", + "read": false, + "rule": 80, + "remote_identifier": "hu4vge" + } +}, +{ + "model": "core.post", + "pk": 3124, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.886Z", + "title": "Double (triple, quadruple...) internet speed with openvpn tap channel bonding to a linux VPS", + "body": "

I have been working a couple of days on my latest video about channel bonding - the video is heavily inspired be this article on Serverfault. In essence, I have been searching for a while on how to bond multiple VPN channels together in order to increase internet speed - there does not seem to be a lot of information around - mainly articles on forums and reddit state that it should be possible but a detailed guide is hard to find. I am using two Ubuntu machines in order to build the connection - one local and one VPS. The bash scripts I use in my video in order to achieve tap channel bonding are available on my github repository. I am currently working on a second video in order to walk through and explain the scripts in depth. Enjoy!

\n\n

(EDIT) - the question has come up in the discussions below if this is really packet load balancing or rather balancing links only - please see my comment further down - I can confirm that this DOES packet balancing so it does work as described.

\n
", + "author": "onemarcfifty", + "publication_date": "2020-07-19T20:41:40Z", + "url": "https://www.reddit.com/r/linux/comments/hu5l4f/double_triple_quadruple_internet_speed_with/", + "read": false, + "rule": 80, + "remote_identifier": "hu5l4f" + } +}, +{ + "model": "core.post", + "pk": 3125, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.888Z", + "title": "OpenRGB - Open source RGB lighting control that doesn't depend on manufacturer software, supports Linux", + "body": "", + "author": "pr0_c0d3", + "publication_date": "2020-07-18T16:52:48Z", + "url": "https://www.reddit.com/r/linux/comments/hthuli/openrgb_open_source_rgb_lighting_control_that/", + "read": false, + "rule": 80, + "remote_identifier": "hthuli" + } +}, +{ + "model": "core.post", + "pk": 3126, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.890Z", + "title": "Make this any sense? Automatic CPU Speed & Power Optimizer", + "body": "", + "author": "spite77", + "publication_date": "2020-07-20T11:53:35Z", + "url": "https://www.reddit.com/r/linux/comments/huikxz/make_this_any_sense_automatic_cpu_speed_power/", + "read": false, + "rule": 80, + "remote_identifier": "huikxz" + } +}, +{ + "model": "core.post", + "pk": 3127, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.891Z", + "title": "Let\u2019s not be pedantic about \u201cOpen Source\u201d", + "body": "", + "author": "speckz", + "publication_date": "2020-07-20T16:46:43Z", + "url": "https://www.reddit.com/r/linux/comments/humirw/lets_not_be_pedantic_about_open_source/", + "read": false, + "rule": 80, + "remote_identifier": "humirw" + } +}, +{ + "model": "core.post", + "pk": 3128, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.893Z", + "title": "Experiences with running Linux Lite", + "body": "", + "author": "daemonpenguin", + "publication_date": "2020-07-20T02:43:49Z", + "url": "https://www.reddit.com/r/linux/comments/hubonw/experiences_with_running_linux_lite/", + "read": false, + "rule": 80, + "remote_identifier": "hubonw" + } +}, +{ + "model": "core.post", + "pk": 3129, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.895Z", + "title": "Tried gnome on arch, surprised how lean it is (used flameshot so it used about 72mb more) closing at 600 megs) on fedora and pop i had gnome eating up 1.3gigs at boot up.", + "body": "
\"Tried
", + "author": "V1n0dKr1shna", + "publication_date": "2020-07-18T13:54:55Z", + "url": "https://www.reddit.com/r/linux/comments/htfeph/tried_gnome_on_arch_surprised_how_lean_it_is_used/", + "read": false, + "rule": 80, + "remote_identifier": "htfeph" + } +}, +{ + "model": "core.post", + "pk": 3130, + "fields": { + "created": "2020-07-20T19:32:35.849Z", + "modified": "2020-07-20T19:32:35.897Z", + "title": "The Free Software Foundation is holding a Fundraiser, help them reach 200 members", + "body": "", + "author": "Neet-Feet", + "publication_date": "2020-07-18T17:55:30Z", + "url": "https://www.reddit.com/r/linux/comments/htiuyi/the_free_software_foundation_is_holding_a/", + "read": false, + "rule": 80, + "remote_identifier": "htiuyi" + } +}, +{ + "model": "core.post", + "pk": 3131, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.899Z", + "title": "Why is the mindset around Arch so negative?", + "body": "

I love the Linux community as a whole. You can find some of the most creative and imaginative people within most Linux communities. On a whole, Linux users are some of the most helpful and informative people you can encounter. Truly the type to think outside the box and learn new things. It can be very inspirational.

\n\n

If I jumped onto Ubuntu, Fedora, or openSUSE's community I can have a free flowing conversation about Linux, their distribution, and getting help or giving help is so free-flowing and easy. The communities are eager to welcome new people and appreciate folks who contribute.

\n\n

Then you have Arch. I love the OS but dislike the mindset. Asking for help is meat with resistance, giving help can also be punishable, and god forbid you try to have a discussion. But it's not just their core community either. For example, I just discovered Endeavour OS which is built around Arch and after 11 post I'm told to come back in 8 hours. Their subReddit here on Reddit, you have to ask to even make 1 post. There of course is also Manjaro Linux and they too have this gatekeeper mindset, the same can be said for ArcoLinux.

\n\n

What is it about Arch that makes everyone want to be either a control freak or a gatekeeper?

\n\n

I do not see this within the Ubuntu or Fedora or openSUSE communities. As I said, their mindset seems eager and willing to unite and work as a community. Am I the only how has noticed this?

\n
", + "author": "Linux-Is-Best", + "publication_date": "2020-07-18T23:28:12Z", + "url": "https://www.reddit.com/r/linux/comments/htojwk/why_is_the_mindset_around_arch_so_negative/", + "read": false, + "rule": 80, + "remote_identifier": "htojwk" + } +}, +{ + "model": "core.post", + "pk": 3132, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.901Z", + "title": "Using the nstat network statistics command in Linux", + "body": "", + "author": "cronos426", + "publication_date": "2020-07-19T17:55:55Z", + "url": "https://www.reddit.com/r/linux/comments/hu2q6v/using_the_nstat_network_statistics_command_in/", + "read": false, + "rule": 80, + "remote_identifier": "hu2q6v" + } +}, +{ + "model": "core.post", + "pk": 3133, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.903Z", + "title": "Contributing via GitLab Merge Requests", + "body": "", + "author": "ChristophCullmann", + "publication_date": "2020-07-18T20:01:26Z", + "url": "https://www.reddit.com/r/linux/comments/htl05p/contributing_via_gitlab_merge_requests/", + "read": false, + "rule": 80, + "remote_identifier": "htl05p" + } +}, +{ + "model": "core.post", + "pk": 3134, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.905Z", + "title": "OpenMandriva: combines WINE64 and 32 into one package capable of running both binaries, i686 architecture was considered as deprecated. Work is underway on a new Rolling release", + "body": "", + "author": "DamonsLinux", + "publication_date": "2020-07-18T15:02:35Z", + "url": "https://www.reddit.com/r/linux/comments/htg9dj/openmandriva_combines_wine64_and_32_into_one/", + "read": false, + "rule": 80, + "remote_identifier": "htg9dj" + } +}, +{ + "model": "core.post", + "pk": 3135, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.906Z", + "title": "OpenRCT2 Player Survey 2020 - Previous survey shows almost 25% players are linux, please help represent linux in the most recent survey", + "body": "", + "author": "christophski", + "publication_date": "2020-07-18T11:39:06Z", + "url": "https://www.reddit.com/r/linux/comments/htdzuh/openrct2_player_survey_2020_previous_survey_shows/", + "read": false, + "rule": 80, + "remote_identifier": "htdzuh" + } +}, +{ + "model": "core.post", + "pk": 3136, + "fields": { + "created": "2020-07-20T19:32:35.853Z", + "modified": "2020-07-20T19:32:35.908Z", + "title": "This week in KDE: Get New Stuff fixes and more", + "body": "", + "author": "kyentei", + "publication_date": "2020-07-18T10:03:46Z", + "url": "https://www.reddit.com/r/linux/comments/htd1an/this_week_in_kde_get_new_stuff_fixes_and_more/", + "read": false, + "rule": 80, + "remote_identifier": "htd1an" + } +}, +{ + "model": "core.post", + "pk": 3137, + "fields": { + "created": "2020-07-20T19:32:35.857Z", + "modified": "2020-07-20T19:32:35.910Z", + "title": "Blender Runs on Linux Pinephone", + "body": "

I managed to get the desktop version of Blender on the Pinephone, and it works really well except for a few bugs.

\n\n

See my post on r/blender:

\n\n

https://www.reddit.com/r/blender/comments/hsxv27/i_installed_blender_on_a_phone/

\n\n

and r/PINE64official:

\n\n

https://www.reddit.com/r/PINE64official/comments/hsxc33/blender_on_pine_phone_almost_usable/

\n\n

I've tried other desktop programs like Xournal and PPSSPP, their UIs also work well, I'd be able to do even more if OpenGL 3 was working.

\n
", + "author": "InfiniteHawk", + "publication_date": "2020-07-17T22:35:14Z", + "url": "https://www.reddit.com/r/linux/comments/ht3d4k/blender_runs_on_linux_pinephone/", + "read": false, + "rule": 80, + "remote_identifier": "ht3d4k" + } +}, +{ + "model": "core.post", + "pk": 3138, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:18:21.616Z", + "title": "Hrmmm They Need to Fix Throttle Animations in the Sabre", + "body": "
", + "author": "TheBootRanger", + "publication_date": "2020-07-21T13:26:01Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv5omc/hrmmm_they_need_to_fix_throttle_animations_in_the/", + "read": true, + "rule": 82, + "remote_identifier": "hv5omc" + } +}, +{ + "model": "core.post", + "pk": 3139, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:18:49.999Z", + "title": "My first 3.10 landing could have gone better...", + "body": "
", + "author": "KnLfey", + "publication_date": "2020-07-21T16:04:50Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv7w85/my_first_310_landing_could_have_gone_better/", + "read": true, + "rule": 82, + "remote_identifier": "hv7w85" + } +}, +{ + "model": "core.post", + "pk": 3140, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:14:50.439Z", + "title": "How about the Christmas in 3 more years?", + "body": "
\"How
", + "author": "SpleanEater", + "publication_date": "2020-07-21T17:49:22Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv9qy8/how_about_the_christmas_in_3_more_years/", + "read": false, + "rule": 82, + "remote_identifier": "hv9qy8" + } +}, +{ + "model": "core.post", + "pk": 3141, + "fields": { + "created": "2020-07-21T20:14:50.415Z", + "modified": "2020-07-21T20:18:33.532Z", + "title": "Long time Elite Dangerous player. New to star citizen i think im doing great", + "body": "", + "author": "Filblo5", + "publication_date": "2020-07-21T15:33:49Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv7elb/long_time_elite_dangerous_player_new_to_star/", + "read": true, + "rule": 82, + "remote_identifier": "hv7elb" + } +}, +{ + "model": "core.post", + "pk": 3142, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.443Z", + "title": "And we stand by it.", + "body": "
\"And
", + "author": "CyberTill", + "publication_date": "2020-07-21T18:57:48Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvb3wm/and_we_stand_by_it/", + "read": false, + "rule": 82, + "remote_identifier": "hvb3wm" + } +}, +{ + "model": "core.post", + "pk": 3143, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.446Z", + "title": "Nomad", + "body": "
\"Nomad\"
", + "author": "ibracitizen", + "publication_date": "2020-07-21T19:52:24Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvc5h3/nomad/", + "read": false, + "rule": 82, + "remote_identifier": "hvc5h3" + } +}, +{ + "model": "core.post", + "pk": 3144, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.449Z", + "title": "Probably the best screen cap i've ever caught on a whim. 3.5 Arc Corp release. Also a confession: I never pledged. Got a ship with my GPU. I intend to pay my dues.", + "body": "
\"Probably
", + "author": "ScionoicS", + "publication_date": "2020-07-21T20:23:01Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvcqzf/probably_the_best_screen_cap_ive_ever_caught_on_a/", + "read": false, + "rule": 82, + "remote_identifier": "hvcqzf" + } +}, +{ + "model": "core.post", + "pk": 3145, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:14:50.451Z", + "title": "Play to escape the depressing job hunt where I need 10 years experience for a entry level job to find this, only been playing for 1 and a half years :(", + "body": "
\"Play
", + "author": "Albert-III-", + "publication_date": "2020-07-21T12:23:45Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv4z08/play_to_escape_the_depressing_job_hunt_where_i/", + "read": false, + "rule": 82, + "remote_identifier": "hv4z08" + } +}, +{ + "model": "core.post", + "pk": 3146, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:19:00.691Z", + "title": "The void beckons.", + "body": "
", + "author": "HisNameWasHis", + "publication_date": "2020-07-21T14:40:51Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv6nij/the_void_beckons/", + "read": true, + "rule": 82, + "remote_identifier": "hv6nij" + } +}, +{ + "model": "core.post", + "pk": 3147, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:19:05.881Z", + "title": "I made a SC-like Photobash with Soldiers", + "body": "
\"I
", + "author": "IsaacPolar", + "publication_date": "2020-07-21T17:13:39Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv92ri/i_made_a_sclike_photobash_with_soldiers/", + "read": true, + "rule": 82, + "remote_identifier": "hv92ri" + } +}, +{ + "model": "core.post", + "pk": 3148, + "fields": { + "created": "2020-07-21T20:14:50.416Z", + "modified": "2020-07-21T20:19:41.227Z", + "title": "Ocean Shader Improvements", + "body": "
\"Ocean
", + "author": "shoeii", + "publication_date": "2020-07-21T18:41:51Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvasds/ocean_shader_improvements/", + "read": true, + "rule": 82, + "remote_identifier": "hvasds" + } +}, +{ + "model": "core.post", + "pk": 3149, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.459Z", + "title": "As much shit as Star Citizen (rightfully) gets it still does one thing better than any other 'game' I've played", + "body": "

It invokes a real sense of scale, on multiple levels.

\n\n

One could argue that's one of the most important feelings you'd want to capture in any game set in space, but of course it's mostly meaningless if there aren't enough gameplay loops and systems in place to work in tandem with and make the space that's been created interesting, and that's where SC is currently a failure.

\n\n

Even so, I think being able to create that sense of smallness isn't insignificant.

\n\n

You as a pilot are dwarfed by your ship which is itself dwarfed by a larger ship which is itself dwarfed by another, even more massive one which is dwarfed by the space station or hub you're at which is dwarfed by a crater on a moon which is dwarfed by the moon itself which is dwarfed by the planet it orbits which is dwarfed by the sheer vastness of space in between all of those things and that they are, despite the distance, still connected.

\n\n

Getting lost in Lorville (even if it is mostly linear) and knowing it's only a small part of the playable space is a really neat feeling - looking out from the windows of the train up into the sky and knowing you can go there and beyond really makes you feel like there is a whole world (and more) waiting to be explored.

\n\n

I think this is a direct result of having legs and not being locked into the cockpit of your ship - I've played more Elite: Dangerous than Star Citizen and it accomplishes a similar sense of scale but, at least not as far as I've felt, never to the same degree - because you're locked in your ship you never really get this same sense of being small or insignificant even though you are dwarfed in similar ways by planets/asteroids/other ships - will be interesting to see how their implementation of 'space legs' in the upcoming expansion changes this.

\n\n

My favourite thing to do in Star Citizen (because there isn't a whole lot) is to just find some pocket of space far away from anything else and just walk around my ship, feeling truly alone and insignificant, gazing out at the void that stretches infinitely all around - something about that is super comfy.

\n\n

I can't think of many other game that accomplish a similar level of scale though I'm sure they exist.

\n\n

I've been playing an indie game called Empyrion - Galactic Survival and it actually is sort of similar to SC in this regard but it's nowhere near as polished or smooth - transitions from atmosphere to space are not truly seamless and planets themselves are kind of stitched together, but it still manages to invoke that same kind of awe at the scale of things when you dock a small vessel to a capital vessel, for example - definitely worth checking out if you like sci-fi/space games, which you must if you're here, but just be prepared for the jank.

\n
", + "author": "thegreatself", + "publication_date": "2020-07-21T20:30:15Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvcw38/as_much_shit_as_star_citizen_rightfully_gets_it/", + "read": false, + "rule": 82, + "remote_identifier": "hvcw38" + } +}, +{ + "model": "core.post", + "pk": 3150, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.462Z", + "title": "You waiting for patch 3.10 to go live while watching tons of videos about the new flight model features. Be patient, 3.11 and 3.12 will be even better.", + "body": "
\"You
", + "author": "jsabater76", + "publication_date": "2020-07-21T09:39:27Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv372v/you_waiting_for_patch_310_to_go_live_while/", + "read": false, + "rule": 82, + "remote_identifier": "hv372v" + } +}, +{ + "model": "core.post", + "pk": 3151, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.466Z", + "title": "CIG, can we please fix these \"black hole\" doors(when they are closed) on ships please.", + "body": "
\"CIG,
", + "author": "AbnormallyBendPenis", + "publication_date": "2020-07-21T13:40:14Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv5uzj/cig_can_we_please_fix_these_black_hole_doorswhen/", + "read": false, + "rule": 82, + "remote_identifier": "hv5uzj" + } +}, +{ + "model": "core.post", + "pk": 3152, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.468Z", + "title": "Anvil Super Hornet over Cellin", + "body": "
\"Anvil
", + "author": "SaraCaterina", + "publication_date": "2020-07-21T20:33:58Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvcyq6/anvil_super_hornet_over_cellin/", + "read": false, + "rule": 82, + "remote_identifier": "hvcyq6" + } +}, +{ + "model": "core.post", + "pk": 3153, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.471Z", + "title": "3.10 Combat Changes", + "body": "", + "author": "STLYoungblood", + "publication_date": "2020-07-21T16:37:44Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv8fr7/310_combat_changes/", + "read": false, + "rule": 82, + "remote_identifier": "hv8fr7" + } +}, +{ + "model": "core.post", + "pk": 3154, + "fields": { + "created": "2020-07-21T20:14:50.420Z", + "modified": "2020-07-21T20:14:50.472Z", + "title": "Hey CIG how about that S42 Vi.... Oh...", + "body": "
\"Hey
", + "author": "SiEDeN", + "publication_date": "2020-07-21T21:37:16Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hve6am/hey_cig_how_about_that_s42_vi_oh/", + "read": false, + "rule": 82, + "remote_identifier": "hve6am" + } +}, +{ + "model": "core.post", + "pk": 3155, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.475Z", + "title": "3.10 M PTU Eclipse improvements", + "body": "

If this goes live, CIG had addressed 2 of my Eclipse critics.

\n\n

Not because of my videos of course, CIG doesn't know I exist.

\n\n

 

\n\n

a. Eclipse has armor stealth in 3.10, see my table:\nhttps://docs.google.com/spreadsheets/d/1OJXg7MQsG_IVTPsmlmZYaxEPK4n4iqnhQx4oigIlJHg/edit#gid=343807746

\n\n

 

\n\n

b. Eclipse can fire her size 9 torpedoes way quicker now, see my video with a side by side comparison of the max firing speed in 3.9 and 3.10:\nhttps://youtu.be/GFTF1Qt7T3o?t=207

\n
", + "author": "Camural", + "publication_date": "2020-07-21T18:15:50Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hva9lc/310_m_ptu_eclipse_improvements/", + "read": false, + "rule": 82, + "remote_identifier": "hva9lc" + } +}, +{ + "model": "core.post", + "pk": 3156, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.477Z", + "title": "Hark! The Drake Herald Sings", + "body": "
\"Hark!
", + "author": "CyrexStorm", + "publication_date": "2020-07-21T16:19:31Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv84kk/hark_the_drake_herald_sings/", + "read": false, + "rule": 82, + "remote_identifier": "hv84kk" + } +}, +{ + "model": "core.post", + "pk": 3157, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.479Z", + "title": "The new flight stick in the Prowler", + "body": "
\"The
", + "author": "Potato_Nades", + "publication_date": "2020-07-21T16:22:22Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv86c2/the_new_flight_stick_in_the_prowler/", + "read": false, + "rule": 82, + "remote_identifier": "hv86c2" + } +}, +{ + "model": "core.post", + "pk": 3158, + "fields": { + "created": "2020-07-21T20:14:50.422Z", + "modified": "2020-07-21T20:14:50.481Z", + "title": "Norwegian VAT charged from August 1st", + "body": "
\"Norwegian
", + "author": "norgeek", + "publication_date": "2020-07-21T10:30:57Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv3r3l/norwegian_vat_charged_from_august_1st/", + "read": false, + "rule": 82, + "remote_identifier": "hv3r3l" + } +}, +{ + "model": "core.post", + "pk": 3159, + "fields": { + "created": "2020-07-21T20:14:50.423Z", + "modified": "2020-07-21T20:14:50.484Z", + "title": "With Pyro (currently WIP), Nyx (partially done), Odin (S42), currently on the way, what is everyone\u2019s thoughts on Terra possibly being next on the list of star systems to be added into the PU within \u2026", + "body": "
\"With
", + "author": "realCLTotaku", + "publication_date": "2020-07-21T13:27:09Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hv5p41/with_pyro_currently_wip_nyx_partially_done_odin/", + "read": false, + "rule": 82, + "remote_identifier": "hv5p41" + } +}, +{ + "model": "core.post", + "pk": 3160, + "fields": { + "created": "2020-07-21T20:14:50.423Z", + "modified": "2020-07-21T20:14:50.486Z", + "title": "Testing out the new electron rifle", + "body": "
", + "author": "joshbaker2112", + "publication_date": "2020-07-21T02:56:19Z", + "url": "https://www.reddit.com/r/starcitizen/comments/huxr6d/testing_out_the_new_electron_rifle/", + "read": false, + "rule": 82, + "remote_identifier": "huxr6d" + } +}, +{ + "model": "core.post", + "pk": 3161, + "fields": { + "created": "2020-07-21T20:14:50.423Z", + "modified": "2020-07-21T20:14:50.487Z", + "title": "Imperial Geographic's Lovecraftian magazine special is here. \ud83d\udc19 Find the link in the comments!", + "body": "
\"Imperial
", + "author": "Good_Punk2", + "publication_date": "2020-07-21T18:21:38Z", + "url": "https://www.reddit.com/r/starcitizen/comments/hvadrh/imperial_geographics_lovecraftian_magazine/", + "read": false, + "rule": 82, + "remote_identifier": "hvadrh" + } +}, +{ + "model": "core.post", + "pk": 3162, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.525Z", + "title": "Linux Distributions Timeline", + "body": "
\"Linux
", + "author": "bauripalash", + "publication_date": "2020-07-21T06:07:59Z", + "url": "https://www.reddit.com/r/linux/comments/hv0ktn/linux_distributions_timeline/", + "read": false, + "rule": 80, + "remote_identifier": "hv0ktn" + } +}, +{ + "model": "core.post", + "pk": 3163, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.527Z", + "title": "Fedora: Proposal to replace default wined3d backend with DXVK", + "body": "", + "author": "friskfrugt", + "publication_date": "2020-07-21T19:42:49Z", + "url": "https://www.reddit.com/r/linux/comments/hvbyyr/fedora_proposal_to_replace_default_wined3d/", + "read": false, + "rule": 80, + "remote_identifier": "hvbyyr" + } +}, +{ + "model": "core.post", + "pk": 3164, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.531Z", + "title": "Update on marketing and communication plans for the LibreOffice 7.x series", + "body": "", + "author": "TheQuantumZero", + "publication_date": "2020-07-21T09:59:23Z", + "url": "https://www.reddit.com/r/linux/comments/hv3erm/update_on_marketing_and_communication_plans_for/", + "read": false, + "rule": 80, + "remote_identifier": "hv3erm" + } +}, +{ + "model": "core.post", + "pk": 3165, + "fields": { + "created": "2020-07-21T20:14:50.497Z", + "modified": "2020-07-21T20:14:50.533Z", + "title": "FOSS job opening: LibreOffice Development Mentor at The Document Foundation", + "body": "", + "author": "themikeosguy", + "publication_date": "2020-07-21T14:26:36Z", + "url": "https://www.reddit.com/r/linux/comments/hv6gfw/foss_job_opening_libreoffice_development_mentor/", + "read": false, + "rule": 80, + "remote_identifier": "hv6gfw" + } +}, +{ + "model": "core.post", + "pk": 3166, + "fields": { + "created": "2020-07-21T20:14:50.503Z", + "modified": "2020-07-21T20:14:50.536Z", + "title": "gomd - quickly display formatted markdown files with code highlight in your browser", + "body": "

Hi all!

\n\n

I wanted to share a project I've been working on recently. I think it reached a stage where it's pretty usable and should work out of the box. gomd sets up a HTTP server and serves a directory in your browser so you can quickly view your markdown files. It comes with some neat features like:

\n\n
    \n
  • Monitoring files - it will monitor files for changes and reload them whenever needed
    • \n
  • Hot reloading - whenever the file you are currently viewing changes, the tab in your browser will reload automatically.
    • \n
  • Code Highlight - All blocks of code in most common languages will be color highlighted.
    • \n
  • Themes - choose from multiple themes like: solarized, monokai, github, dracula...
    • \n
\n\n

Link: gomd

\n\n

For now its only available from AUR or built from source.

\n\n

​

\n\n

Any tips or feedback will be greatly appreciated :)

\n
", + "author": "wwojtekk", + "publication_date": "2020-07-21T20:07:31Z", + "url": "https://www.reddit.com/r/linux/comments/hvcg44/gomd_quickly_display_formatted_markdown_files/", + "read": false, + "rule": 80, + "remote_identifier": "hvcg44" + } +}, +{ + "model": "core.post", + "pk": 3167, + "fields": { + "created": "2020-07-21T20:14:50.503Z", + "modified": "2020-07-21T20:14:50.543Z", + "title": "They're not otherwise wrong, but it didn't become a real Internet standard until 2017.", + "body": "
\"They're
", + "author": "foodown", + "publication_date": "2020-07-21T21:39:09Z", + "url": "https://www.reddit.com/r/linux/comments/hve7l5/theyre_not_otherwise_wrong_but_it_didnt_become_a/", + "read": false, + "rule": 80, + "remote_identifier": "hve7l5" + } +}, +{ + "model": "core.post", + "pk": 3168, + "fields": { + "created": "2020-07-21T20:14:50.503Z", + "modified": "2020-07-21T20:14:50.545Z", + "title": "Drawing - an alternative to Paint for Linux (gtk3, support HiDPI)", + "body": "", + "author": "dontdieych", + "publication_date": "2020-07-21T02:37:22Z", + "url": "https://www.reddit.com/r/linux/comments/huxgsg/drawing_an_alternative_to_paint_for_linux_gtk3/", + "read": false, + "rule": 80, + "remote_identifier": "huxgsg" + } +}, +{ + "model": "core.post", + "pk": 3169, + "fields": { + "created": "2020-07-21T20:14:50.509Z", + "modified": "2020-07-21T20:14:50.547Z", + "title": "Observations on a Linux issue with 3.5mm earphones with a mic", + "body": "

Alright hello. I have come from r/SolusProject and I made a post there to do with headphone issues. I suggest you read through the post and comments to get a better understanding before reading this https://www.reddit.com/r/SolusProject/comments/hsql4d/frustrating_headphone_issues/. I had posted to do with it again, but it got taken down for duplication (when it wasn't duplication). This post is more of my observations from experimenting and such. There are distros I haven't tried but I tried a wide range of distros like manjaro, ubuntu based ones and all solus flavors, and I was looking more for how well they worked out of the box, rather than with fiddling around with pulse, hdajack etc which I know will work eventually. If you stumbled across this from searching about the same issue I have (or similar) or are confused to what this is about, I suggest you look at my previous post also.

\n\n

So anyways, I've tried the past few days mounting isos to usb drives and trying live os and installing various distros to see about the headphone issue. And my conclusion is that this issue affects the linux kernel in some way across the board. I don't really understand why completely but I have some kind of idea.

\n\n

From installing fresh distros, I noticed that the earphones (they are 3.5mm earphones + mic) get recognised as a microphone and not as a speaker system of some kind. Every single time I had a look at the sound settings and in pulse, they came up as plugged microphone, with the internal speakers being the only output device every single time. It's really odd seeing as how ubuntu 14.04 and xubuntu etc from years past worked flawlessly with the earphones, even manjaro a while ago on my older craptop worked fine. I don't really understand why it doesn't work on my device now.

\n\n

I'll leave my specs at the bottom of this post but what I think is is there's something the manufacturer did, or something like the cpu causes issue with linux. The manufacturer of my laptop is Lenovo, and the cpu/igpu is from AMD. A warning sign is that when installing a linux distro, it doesn't bring up the dual boot menu at startup like it should. Instead it completely hides the fact it exists until I use something like easyuefi to add an option for that distro, how it works is you specify the boot partition, whether it's linux or windows and the loader conf file for the distro. All of this hassle everytime doesn't appear on my craptop, because the dual boot menu appears flawlessly without issue. May be because it uses an Intel cpu/igpu unlike my newer laptop but it's hard to say.

\n\n

Also, it seems like the devices that appear in a given distro when looking at alsa, is hd generic devices but by reloading alsa or any command that shows the full name of the device, it says it's Intel. I don't know if that would be an issue, maybe amd use intel sound drivers or something. It's odd nonetheless.

\n\n

This issue has been boggling my mind for obvious reasons, with half-rhetorical questions like does linux not support the earphones anymore, whether out of accident from an overlooked bug in an update or intentionally phasing out? Is any of this AMD or Lenovo's fault? Even with proper headphones or something, will they fail? I don't think anyone here really knows, hell I'd bet an extreme that no one really understands why in the linux community. I kinda rambled in this post with stuff that should've been said in the last post/thread, but I'm saying it now.

\n\n

Thanks for contributing thus far to this discussion in figuring this out.

\n\n

Specs: AMD Ryzen 5 3500U Mobile CPU (2.2 - 3.7ghz quad core)

\n\n

Radeon Vega 8 Integrated GPU, 8GB Ram, 256GB SSD.

\n\n

Lenovo C340-14API Laptop

\n
", + "author": "BrianMeerkatlol", + "publication_date": "2020-07-21T21:02:19Z", + "url": "https://www.reddit.com/r/linux/comments/hvdi3o/observations_on_a_linux_issue_with_35mm_earphones/", + "read": false, + "rule": 80, + "remote_identifier": "hvdi3o" + } +}, +{ + "model": "core.post", + "pk": 3170, + "fields": { + "created": "2020-07-21T20:14:50.509Z", + "modified": "2020-07-21T20:14:50.549Z", + "title": "South Korean distro HamoniKR OS has been added to Distrowatch", + "body": "", + "author": "TheHordeRisesAgain", + "publication_date": "2020-07-21T07:44:21Z", + "url": "https://www.reddit.com/r/linux/comments/hv1ug1/south_korean_distro_hamonikr_os_has_been_added_to/", + "read": false, + "rule": 80, + "remote_identifier": "hv1ug1" + } +}, +{ + "model": "core.post", + "pk": 3171, + "fields": { + "created": "2020-07-21T20:14:50.509Z", + "modified": "2020-07-21T20:14:50.559Z", + "title": "The Jailer is free! New release of the outstanding database subsetter and browser is available.", + "body": "", + "author": "Plane-Discussion", + "publication_date": "2020-07-21T12:53:54Z", + "url": "https://www.reddit.com/r/linux/comments/hv5b0j/the_jailer_is_free_new_release_of_the_outstanding/", + "read": false, + "rule": 80, + "remote_identifier": "hv5b0j" + } +}, +{ + "model": "core.post", + "pk": 3172, + "fields": { + "created": "2020-07-21T20:14:50.513Z", + "modified": "2020-07-21T20:14:50.563Z", + "title": "A few very well-aged excerpts from Microsoft\u2019s infamous 2004 \u201cGet the facts\u201d campaign, where they make the case for Windows servers being cheaper, more secure, and more performant than Linux servers", + "body": "
\n

Get the facts on Windows and Linux.

\n\n

Leading companies and third-party analysts confirm it: Windows has a lower total cost of ownership and outperforms Linux.

\n\n

...

\n\n

-Security

\n\n

Windows Users Have Fewer Vulnerabilities

\n
\n\n

And then literally the very next bullet point:

\n\n
\n

-Featured Customer Case Study

\n\n

Equifax

\n\n

Equifax Sees 14 Percent Cost Savings

\n\n

Find out why Equifax, a global leader in transforming data into intelligence, selected Windows over Linux to enhance the speed and performance of its marketing services capabilities. Using Microsoft Windows Server System, the company has seen 14 percent in cost savings over Linux.

\n
\n\n

Good thing they saved 14% and got all that extra security! Sure their website is janky and their login flow is downright horrifying (Check it out if you want to be amazed), but who could blame them? Linux is \u201cProhibitively Expensive, Extremely Complex, and Provides No Tangible Business Gains\u201d, Microsoft said so!

\n\n

Source: https://web.archive.org/web/20041027003759/http://www.microsoft.com/windowsserversystem/facts/default.mspx

\n
", + "author": "kevinhaze", + "publication_date": "2020-07-20T21:42:15Z", + "url": "https://www.reddit.com/r/linux/comments/hus5lz/a_few_very_wellaged_excerpts_from_microsofts/", + "read": false, + "rule": 80, + "remote_identifier": "hus5lz" + } +}, +{ + "model": "core.post", + "pk": 3173, + "fields": { + "created": "2020-07-21T20:14:50.515Z", + "modified": "2020-07-21T20:14:50.566Z", + "title": "Are there are any professional audio recording studios or artists that use Linux?", + "body": "

As the title says, who is using Linux as a professional audio engineer, producer, or artist? I am a former Mac user myself, and I am seeing people from time to time who have become disillusioned with what Apple has been doing for the past few years. However, I'm not sure if Linux really has a place for these people to land if they are serious about what they do.

\n\n

Fedora Design Suite and Ubuntu Studio are definitely encouraging to see, but what is their real-world usage like? Are we getting better with professional audio in Linux, or have things been stagnant for years?

\n
", + "author": "RootHouston", + "publication_date": "2020-07-21T00:08:26Z", + "url": "https://www.reddit.com/r/linux/comments/huuxvq/are_there_are_any_professional_audio_recording/", + "read": false, + "rule": 80, + "remote_identifier": "huuxvq" + } +}, +{ + "model": "core.post", + "pk": 3174, + "fields": { + "created": "2020-07-21T20:14:50.515Z", + "modified": "2020-07-21T20:14:50.570Z", + "title": "When Linux had marketing", + "body": "", + "author": "Commodore256", + "publication_date": "2020-07-21T14:03:56Z", + "url": "https://www.reddit.com/r/linux/comments/hv65oa/when_linux_had_marketing/", + "read": false, + "rule": 80, + "remote_identifier": "hv65oa" + } +}, +{ + "model": "core.post", + "pk": 3175, + "fields": { + "created": "2020-07-21T20:14:50.520Z", + "modified": "2020-07-21T20:14:50.598Z", + "title": "Ward: Simple and minimalistic server dashboard", + "body": "

Ward is a simple and and minimalistic server monitoring tool. Ward supports adaptive design system. Also it supports dark theme. It shows only principal information and can be used, if you want to see nice looking dashboard instead looking on bunch of numbers and graphs. Ward works nice on all popular operating systems, because it uses OSHI.

\n\n

https://preview.redd.it/gdppswc3a3c51.png?width=1448&format=png&auto=webp&s=0d6e10146c105ddcfd045dd59c970d4c127ddb8c

\n\n

https://github.com/B-Software/Ward

\n
", + "author": "Pabyzu", + "publication_date": "2020-07-21T00:33:40Z", + "url": "https://www.reddit.com/r/linux/comments/huvea3/ward_simple_and_minimalistic_server_dashboard/", + "read": false, + "rule": 80, + "remote_identifier": "huvea3" + } +}, +{ + "model": "core.post", + "pk": 3176, + "fields": { + "created": "2020-07-21T20:14:50.522Z", + "modified": "2020-07-21T20:14:50.606Z", + "title": "WindowsFX - a good Windows alternative?", + "body": "

I would personally like to hear some of your opinions (in the replies) about WindowsFX. What is WindowsFX you may ask? WindowsFX is a Brazilian linux distribution that is designed to look and act like Windows 10.

\n\n

Linux / WindowsFX is based off of Ubuntu, and uses Cinnamon as its DE. Upon first boot, normal Windows users can tell the difference. But if you were to put it in front of a non tech-savvy person, they wouldn't be able to tell the difference.

\n\n

Personally, with WSL on Windows, I see no need for a distro like this. However, as I said, I would like to hear your opinions on this distro.

\n\n

Video review here.

\n
", + "author": "Demonitized101", + "publication_date": "2020-07-20T23:03:29Z", + "url": "https://www.reddit.com/r/linux/comments/hutpt5/windowsfx_a_good_windows_alternative/", + "read": false, + "rule": 80, + "remote_identifier": "hutpt5" + } +}, +{ + "model": "core.post", + "pk": 3177, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:14:50.780Z", + "title": "Every day this good boy brings a carrot to his best buddy", + "body": "
", + "author": "TooShiftyForYou", + "publication_date": "2020-07-21T15:25:31Z", + "url": "https://www.reddit.com/r/aww/comments/hv7a8b/every_day_this_good_boy_brings_a_carrot_to_his/", + "read": false, + "rule": 81, + "remote_identifier": "hv7a8b" + } +}, +{ + "model": "core.post", + "pk": 3178, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-25T20:08:34.264Z", + "title": "Kitten mimics his human petting the dog", + "body": "
", + "author": "SpecterAscendant", + "publication_date": "2020-07-21T14:56:57Z", + "url": "https://www.reddit.com/r/aww/comments/hv6ve3/kitten_mimics_his_human_petting_the_dog/", + "read": true, + "rule": 81, + "remote_identifier": "hv6ve3" + } +}, +{ + "model": "core.post", + "pk": 3179, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:14:50.789Z", + "title": "My fox friend!", + "body": "
", + "author": "Zepantha", + "publication_date": "2020-07-21T14:27:25Z", + "url": "https://www.reddit.com/r/aww/comments/hv6gte/my_fox_friend/", + "read": false, + "rule": 81, + "remote_identifier": "hv6gte" + } +}, +{ + "model": "core.post", + "pk": 3180, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:15:46.876Z", + "title": "Ducks annihilate peas", + "body": "
", + "author": "tommycalibre", + "publication_date": "2020-07-21T17:12:40Z", + "url": "https://www.reddit.com/r/aww/comments/hv9258/ducks_annihilate_peas/", + "read": true, + "rule": 81, + "remote_identifier": "hv9258" + } +}, +{ + "model": "core.post", + "pk": 3181, + "fields": { + "created": "2020-07-21T20:14:50.775Z", + "modified": "2020-07-21T20:14:50.797Z", + "title": "Wiggle it baby", + "body": "
", + "author": "neo_star", + "publication_date": "2020-07-21T18:44:31Z", + "url": "https://www.reddit.com/r/aww/comments/hvaucy/wiggle_it_baby/", + "read": false, + "rule": 81, + "remote_identifier": "hvaucy" + } +}, +{ + "model": "core.post", + "pk": 3182, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:16:22.725Z", + "title": "I guess I should do this.. everyone seems to be liking little pups and kittens so.. Reddit, meet bailey", + "body": "
\"I
", + "author": "X_XNOTHINGX_X", + "publication_date": "2020-07-21T14:15:08Z", + "url": "https://www.reddit.com/r/aww/comments/hv6b0a/i_guess_i_should_do_this_everyone_seems_to_be/", + "read": true, + "rule": 81, + "remote_identifier": "hv6b0a" + } +}, +{ + "model": "core.post", + "pk": 3183, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.806Z", + "title": "The hat makes the crab.", + "body": "
\"The
", + "author": "fujfuj", + "publication_date": "2020-07-21T14:48:40Z", + "url": "https://www.reddit.com/r/aww/comments/hv6rde/the_hat_makes_the_crab/", + "read": false, + "rule": 81, + "remote_identifier": "hv6rde" + } +}, +{ + "model": "core.post", + "pk": 3184, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.812Z", + "title": "Baby bunny fits in hand", + "body": "
", + "author": "Hawken10", + "publication_date": "2020-07-21T12:31:30Z", + "url": "https://www.reddit.com/r/aww/comments/hv5253/baby_bunny_fits_in_hand/", + "read": false, + "rule": 81, + "remote_identifier": "hv5253" + } +}, +{ + "model": "core.post", + "pk": 3185, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.818Z", + "title": "My cat and I, both pregnant", + "body": "
\"My
", + "author": "nixdionisio", + "publication_date": "2020-07-21T11:06:25Z", + "url": "https://www.reddit.com/r/aww/comments/hv44m2/my_cat_and_i_both_pregnant/", + "read": false, + "rule": 81, + "remote_identifier": "hv44m2" + } +}, +{ + "model": "core.post", + "pk": 3186, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.822Z", + "title": "Very sweet dance", + "body": "
", + "author": "Ashley1023", + "publication_date": "2020-07-21T13:03:03Z", + "url": "https://www.reddit.com/r/aww/comments/hv5ewq/very_sweet_dance/", + "read": false, + "rule": 81, + "remote_identifier": "hv5ewq" + } +}, +{ + "model": "core.post", + "pk": 3187, + "fields": { + "created": "2020-07-21T20:14:50.776Z", + "modified": "2020-07-21T20:14:50.825Z", + "title": "My local pet-store has a cat named Vegemite \u2764\ufe0f", + "body": "
\"My
", + "author": "galinhad", + "publication_date": "2020-07-21T12:06:17Z", + "url": "https://www.reddit.com/r/aww/comments/hv4s5z/my_local_petstore_has_a_cat_named_vegemite/", + "read": false, + "rule": 81, + "remote_identifier": "hv4s5z" + } +}, +{ + "model": "core.post", + "pk": 3188, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-21T20:15:01.459Z", + "title": "A teacher like that makes a huge difference", + "body": "
", + "author": "Unicornglitteryblood", + "publication_date": "2020-07-21T18:29:57Z", + "url": "https://www.reddit.com/r/aww/comments/hvajo9/a_teacher_like_that_makes_a_huge_difference/", + "read": true, + "rule": 81, + "remote_identifier": "hvajo9" + } +}, +{ + "model": "core.post", + "pk": 3189, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-22T19:55:49.930Z", + "title": "Kitten Encounters Bubbly Water", + "body": "
\"Kitten
", + "author": "DragonOBunny", + "publication_date": "2020-07-21T15:28:05Z", + "url": "https://www.reddit.com/r/aww/comments/hv7bis/kitten_encounters_bubbly_water/", + "read": true, + "rule": 81, + "remote_identifier": "hv7bis" + } +}, +{ + "model": "core.post", + "pk": 3190, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-21T20:14:50.833Z", + "title": "Are These My Chickens Now?", + "body": "", + "author": "jasontaken", + "publication_date": "2020-07-21T09:55:36Z", + "url": "https://www.reddit.com/r/aww/comments/hv3de1/are_these_my_chickens_now/", + "read": false, + "rule": 81, + "remote_identifier": "hv3de1" + } +}, +{ + "model": "core.post", + "pk": 3191, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-25T20:08:20.518Z", + "title": "Our St Bernard 6 months apart", + "body": "
\"Our
", + "author": "ryan3105", + "publication_date": "2020-07-21T18:00:04Z", + "url": "https://www.reddit.com/r/aww/comments/hv9yea/our_st_bernard_6_months_apart/", + "read": true, + "rule": 81, + "remote_identifier": "hv9yea" + } +}, +{ + "model": "core.post", + "pk": 3192, + "fields": { + "created": "2020-07-21T20:14:50.777Z", + "modified": "2020-07-21T20:14:50.837Z", + "title": "Father and child in sync", + "body": "
", + "author": "Araragi_Monogatari", + "publication_date": "2020-07-21T08:29:18Z", + "url": "https://www.reddit.com/r/aww/comments/hv2enj/father_and_child_in_sync/", + "read": false, + "rule": 81, + "remote_identifier": "hv2enj" + } +}, +{ + "model": "core.post", + "pk": 3193, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.840Z", + "title": "A meme is born", + "body": "
\"A
", + "author": "Unicornglitteryblood", + "publication_date": "2020-07-21T18:55:04Z", + "url": "https://www.reddit.com/r/aww/comments/hvb1vh/a_meme_is_born/", + "read": false, + "rule": 81, + "remote_identifier": "hvb1vh" + } +}, +{ + "model": "core.post", + "pk": 3194, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.842Z", + "title": "She bites, then she sleeps, then bites again, then sleeps again. \ud83d\ude02", + "body": "
", + "author": "earlymauvs", + "publication_date": "2020-07-21T11:34:19Z", + "url": "https://www.reddit.com/r/aww/comments/hv4fat/she_bites_then_she_sleeps_then_bites_again_then/", + "read": false, + "rule": 81, + "remote_identifier": "hv4fat" + } +}, +{ + "model": "core.post", + "pk": 3195, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.844Z", + "title": "Nothing calmer that 2 ginger cats rubbing heads and showing their love in morning", + "body": "
\"Nothing
", + "author": "Apotheosis33", + "publication_date": "2020-07-21T08:39:24Z", + "url": "https://www.reddit.com/r/aww/comments/hv2j2g/nothing_calmer_that_2_ginger_cats_rubbing_heads/", + "read": false, + "rule": 81, + "remote_identifier": "hv2j2g" + } +}, +{ + "model": "core.post", + "pk": 3196, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.851Z", + "title": "Ring Tailed Possum", + "body": "", + "author": "Wayward-Delver", + "publication_date": "2020-07-21T11:23:51Z", + "url": "https://www.reddit.com/r/aww/comments/hv4b9e/ring_tailed_possum/", + "read": false, + "rule": 81, + "remote_identifier": "hv4b9e" + } +}, +{ + "model": "core.post", + "pk": 3197, + "fields": { + "created": "2020-07-21T20:14:50.778Z", + "modified": "2020-07-21T20:14:50.854Z", + "title": "Baby scooby in sad mood....", + "body": "
\"Baby
", + "author": "deepanshuahiroo7", + "publication_date": "2020-07-21T15:12:23Z", + "url": "https://www.reddit.com/r/aww/comments/hv73ft/baby_scooby_in_sad_mood/", + "read": false, + "rule": 81, + "remote_identifier": "hv73ft" + } +}, +{ + "model": "core.post", + "pk": 3198, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:14:50.856Z", + "title": "New friends!", + "body": "
\"New
", + "author": "HelentotheKeller", + "publication_date": "2020-07-21T13:10:48Z", + "url": "https://www.reddit.com/r/aww/comments/hv5i6i/new_friends/", + "read": false, + "rule": 81, + "remote_identifier": "hv5i6i" + } +}, +{ + "model": "core.post", + "pk": 3199, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:14:50.858Z", + "title": "When you haven't chewed anything for 1 second", + "body": "
\"When
", + "author": "Tanay4", + "publication_date": "2020-07-21T10:26:53Z", + "url": "https://www.reddit.com/r/aww/comments/hv3pl0/when_you_havent_chewed_anything_for_1_second/", + "read": false, + "rule": 81, + "remote_identifier": "hv3pl0" + } +}, +{ + "model": "core.post", + "pk": 3200, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:17:01.490Z", + "title": "Mango Derp", + "body": "
\"Mango
", + "author": "sheetglass", + "publication_date": "2020-07-21T13:27:26Z", + "url": "https://www.reddit.com/r/aww/comments/hv5p8s/mango_derp/", + "read": true, + "rule": 81, + "remote_identifier": "hv5p8s" + } +}, +{ + "model": "core.post", + "pk": 3201, + "fields": { + "created": "2020-07-21T20:14:50.779Z", + "modified": "2020-07-21T20:14:50.863Z", + "title": "My guy turns 20 next month", + "body": "
\"My
", + "author": "alozsoc", + "publication_date": "2020-07-21T06:34:26Z", + "url": "https://www.reddit.com/r/aww/comments/hv0xp1/my_guy_turns_20_next_month/", + "read": false, + "rule": 81, + "remote_identifier": "hv0xp1" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "add_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "change_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "delete_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view log entry", + "content_type": [ + "admin", + "logentry" + ], + "codename": "view_logentry" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "add_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "change_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "delete_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view permission", + "content_type": [ + "auth", + "permission" + ], + "codename": "view_permission" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add group", + "content_type": [ + "auth", + "group" + ], + "codename": "add_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change group", + "content_type": [ + "auth", + "group" + ], + "codename": "change_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete group", + "content_type": [ + "auth", + "group" + ], + "codename": "delete_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view group", + "content_type": [ + "auth", + "group" + ], + "codename": "view_group" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "add_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "change_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "delete_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view content type", + "content_type": [ + "contenttypes", + "contenttype" + ], + "codename": "view_contenttype" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add session", + "content_type": [ + "sessions", + "session" + ], + "codename": "add_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change session", + "content_type": [ + "sessions", + "session" + ], + "codename": "change_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete session", + "content_type": [ + "sessions", + "session" + ], + "codename": "delete_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view session", + "content_type": [ + "sessions", + "session" + ], + "codename": "view_session" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "add_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "change_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "delete_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view crontab", + "content_type": [ + "django_celery_beat", + "crontabschedule" + ], + "codename": "view_crontabschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "add_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "change_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "delete_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view interval", + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "codename": "view_intervalschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "add_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "change_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "delete_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view periodic task", + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "codename": "view_periodictask" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "add_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "change_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "delete_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view periodic tasks", + "content_type": [ + "django_celery_beat", + "periodictasks" + ], + "codename": "view_periodictasks" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "add_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "change_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "delete_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view solar event", + "content_type": [ + "django_celery_beat", + "solarschedule" + ], + "codename": "view_solarschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "add_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "change_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "delete_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view clocked", + "content_type": [ + "django_celery_beat", + "clockedschedule" + ], + "codename": "view_clockedschedule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "add_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "change_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "delete_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view registration profile", + "content_type": [ + "registration", + "registrationprofile" + ], + "codename": "view_registrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "add_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "change_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "delete_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view supervised registration profile", + "content_type": [ + "registration", + "supervisedregistrationprofile" + ], + "codename": "view_supervisedregistrationprofile" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "add_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "change_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "delete_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view access attempt", + "content_type": [ + "axes", + "accessattempt" + ], + "codename": "view_accessattempt" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "add_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "change_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "delete_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view access log", + "content_type": [ + "axes", + "accesslog" + ], + "codename": "view_accesslog" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add user", + "content_type": [ + "accounts", + "user" + ], + "codename": "add_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change user", + "content_type": [ + "accounts", + "user" + ], + "codename": "change_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete user", + "content_type": [ + "accounts", + "user" + ], + "codename": "delete_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view user", + "content_type": [ + "accounts", + "user" + ], + "codename": "view_user" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add post", + "content_type": [ + "core", + "post" + ], + "codename": "add_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change post", + "content_type": [ + "core", + "post" + ], + "codename": "change_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete post", + "content_type": [ + "core", + "post" + ], + "codename": "delete_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view post", + "content_type": [ + "core", + "post" + ], + "codename": "view_post" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add Category", + "content_type": [ + "core", + "category" + ], + "codename": "add_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change Category", + "content_type": [ + "core", + "category" + ], + "codename": "change_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete Category", + "content_type": [ + "core", + "category" + ], + "codename": "delete_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view Category", + "content_type": [ + "core", + "category" + ], + "codename": "view_category" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can add collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "add_collectionrule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can change collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "change_collectionrule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can delete collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "delete_collectionrule" + } +}, +{ + "model": "auth.permission", + "fields": { + "name": "Can view collection rule", + "content_type": [ + "collection", + "collectionrule" + ], + "codename": "view_collectionrule" + } +}, +{ + "model": "accounts.user", + "fields": { + "password": "pbkdf2_sha256$180000$U9a2CS9X0b8Y$T6bD/VoUOFoGNIp16aFlOL0N7q0e6A3I97ypm/AhsGo=", + "last_login": "2020-07-21T20:14:35.966Z", + "is_superuser": true, + "first_name": "", + "last_name": "", + "is_staff": true, + "is_active": true, + "date_joined": "2019-07-18T18:52:36.080Z", + "email": "sonny@bakker.nl", + "task": 10, + "reddit_refresh_token": null, + "reddit_access_token": null, + "groups": [], + "user_permissions": [] + } +}, +{ + "model": "core.category", + "pk": 8, + "fields": { + "created": "2019-11-17T19:37:24.671Z", + "modified": "2019-11-18T19:59:55.010Z", + "name": "World news", + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "core.category", + "pk": 9, + "fields": { + "created": "2019-11-17T19:37:26.161Z", + "modified": "2020-05-30T13:36:10.509Z", + "name": "Tech", + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 3, + "fields": { + "created": "2019-07-14T13:08:10.374Z", + "modified": "2020-07-14T11:45:30.680Z", + "name": "Hackers News", + "type": "feed", + "url": "https://news.ycombinator.com/rss", + "website_url": "https://news.ycombinator.com/", + "favicon": "https://news.ycombinator.com/favicon.ico", + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:30.477Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 4, + "fields": { + "created": "2019-07-20T11:24:32.745Z", + "modified": "2020-07-14T11:45:29.357Z", + "name": "BBC", + "type": "feed", + "url": "http://feeds.bbci.co.uk/news/world/rss.xml", + "website_url": "https://www.bbc.co.uk/news/", + "favicon": "https://m.files.bbci.co.uk/modules/bbc-morph-news-waf-page-meta/2.5.2/apple-touch-icon-57x57-precomposed.png", + "timezone": "UTC", + "category": 8, + "last_run": "2020-07-14T11:45:28.863Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 5, + "fields": { + "created": "2019-07-20T11:24:50.411Z", + "modified": "2020-07-14T11:45:30.063Z", + "name": "Ars Technica", + "type": "feed", + "url": "http://feeds.arstechnica.com/arstechnica/index?fmt=xml", + "website_url": "https://arstechnica.com", + "favicon": "https://cdn.arstechnica.net/favicon.ico", + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:29.810Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 6, + "fields": { + "created": "2019-07-20T11:25:02.089Z", + "modified": "2020-07-14T11:45:30.473Z", + "name": "The Guardian", + "type": "feed", + "url": "https://www.theguardian.com/world/rss", + "website_url": "https://www.theguardian.com/world", + "favicon": "https://assets.guim.co.uk/images/favicons/873381bf11d58e20f551905d51575117/72x72.png", + "timezone": "UTC", + "category": 8, + "last_run": "2020-07-14T11:45:30.181Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 7, + "fields": { + "created": "2019-07-20T11:25:30.121Z", + "modified": "2020-07-14T11:45:29.807Z", + "name": "Tweakers", + "type": "feed", + "url": "http://feeds.feedburner.com/tweakers/mixed?fmt=xml", + "website_url": "https://tweakers.net/", + "favicon": null, + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:29.525Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 8, + "fields": { + "created": "2019-07-20T11:25:46.256Z", + "modified": "2020-07-14T11:45:30.179Z", + "name": "The Verge", + "type": "feed", + "url": "https://www.theverge.com/rss/index.xml", + "website_url": "https://www.theverge.com/", + "favicon": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395367/favicon-16x16.0.png", + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-14T11:45:30.066Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 9, + "fields": { + "created": "2019-11-24T15:28:41.399Z", + "modified": "2020-07-14T11:45:29.522Z", + "name": "NOS", + "type": "feed", + "url": "http://feeds.nos.nl/nosnieuwsalgemeen", + "website_url": null, + "favicon": null, + "timezone": "Europe/Amsterdam", + "category": 8, + "last_run": "2020-07-14T11:45:29.362Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 80, + "fields": { + "created": "2020-07-08T19:30:10.638Z", + "modified": "2020-07-21T20:14:50.609Z", + "name": "Linux subreddit", + "type": "subreddit", + "url": "https://oauth.reddit.com/r/linux/hot", + "website_url": null, + "favicon": null, + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-21T20:14:50.492Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 81, + "fields": { + "created": "2020-07-08T19:30:33.590Z", + "modified": "2020-07-21T20:14:50.865Z", + "name": "AWW subreddit", + "type": "subreddit", + "url": "https://oauth.reddit.com/r/aww/hot", + "website_url": null, + "favicon": null, + "timezone": "UTC", + "category": 8, + "last_run": "2020-07-21T20:14:50.768Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "collection.collectionrule", + "pk": 82, + "fields": { + "created": "2020-07-20T19:29:37.675Z", + "modified": "2020-07-21T20:14:50.489Z", + "name": "Star citizen subreddit", + "type": "subreddit", + "url": "https://oauth.reddit.com/r/starcitizen/hot.json", + "website_url": null, + "favicon": null, + "timezone": "UTC", + "category": 9, + "last_run": "2020-07-21T20:14:50.355Z", + "succeeded": true, + "error": null, + "enabled": true, + "user": [ + "sonny@bakker.nl" + ] + } +}, +{ + "model": "admin.logentry", + "pk": 1, + "fields": { + "action_time": "2020-05-24T18:38:44.624Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "intervalschedule" + ], + "object_id": "5", + "object_repr": "every 4 hours", + "action_flag": 1, + "change_message": "[{\"added\": {}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 2, + "fields": { + "action_time": "2020-05-24T18:38:46.689Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "10", + "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Interval Schedule\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 3, + "fields": { + "action_time": "2020-05-24T18:39:09.203Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "26", + "object_repr": "sonnyba871@gmail.com-collection-task: every hour", + "action_flag": 3, + "change_message": "" + } +}, +{ + "model": "admin.logentry", + "pk": 4, + "fields": { + "action_time": "2020-05-24T19:46:50.248Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "10", + "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Positional Arguments\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 5, + "fields": { + "action_time": "2020-07-07T19:37:57.086Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Reddit refresh token\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 6, + "fields": { + "action_time": "2020-07-07T19:39:46.160Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "10", + "object_repr": "sonny@bakker.nl-collection-task: every 4 hours", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Task (registered)\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 7, + "fields": { + "action_time": "2020-07-08T19:29:27.025Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "django_celery_beat", + "periodictask" + ], + "object_id": "11", + "object_repr": "Reddit collection task: every 4 hours", + "action_flag": 1, + "change_message": "[{\"added\": {}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 8, + "fields": { + "action_time": "2020-07-14T11:46:50.039Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 9, + "fields": { + "action_time": "2020-07-18T19:08:33.997Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "collection", + "collectionrule" + ], + "object_id": "81", + "object_repr": "AWW subreddit", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 10, + "fields": { + "action_time": "2020-07-18T19:08:44.063Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "collection", + "collectionrule" + ], + "object_id": "80", + "object_repr": "Linux subreddit", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Url\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 11, + "fields": { + "action_time": "2020-07-18T19:17:25.213Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2336", + "object_repr": "Post-2336", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 12, + "fields": { + "action_time": "2020-07-18T19:17:40.596Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2336", + "object_repr": "Post-2336", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 13, + "fields": { + "action_time": "2020-07-19T10:55:55.807Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2764", + "object_repr": "Post-2764", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 14, + "fields": { + "action_time": "2020-07-19T10:57:40.643Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2764", + "object_repr": "Post-2764", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 15, + "fields": { + "action_time": "2020-07-19T10:58:05.823Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "core", + "post" + ], + "object_id": "2764", + "object_repr": "Post-2764", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Body\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 16, + "fields": { + "action_time": "2020-07-26T09:51:52.478Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 17, + "fields": { + "action_time": "2020-07-26T09:52:04.691Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"password\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 18, + "fields": { + "action_time": "2020-07-26T09:52:12.392Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"First name\"]}}]" + } +}, +{ + "model": "admin.logentry", + "pk": 19, + "fields": { + "action_time": "2020-07-26T09:56:15.949Z", + "user": [ + "sonny@bakker.nl" + ], + "content_type": [ + "accounts", + "user" + ], + "object_id": "1", + "object_repr": "sonny@bakker.nl", + "action_flag": 2, + "change_message": "[{\"changed\": {\"fields\": [\"Reddit access token\", \"Reddit refresh token\"]}}]" + } +} +] diff --git a/src/newsreader/fixtures/local/fixture.json b/src/newsreader/fixtures/local/fixture.json index ffcc4fd..99176b5 100644 --- a/src/newsreader/fixtures/local/fixture.json +++ b/src/newsreader/fixtures/local/fixture.json @@ -47,7 +47,7 @@ "user" : 2, "succeeded" : true, "modified" : "2019-07-20T11:28:16.473Z", - "last_suceeded" : "2019-07-20T11:28:16.316Z", + "last_run" : "2019-07-20T11:28:16.316Z", "name" : "Hackers News", "website_url" : null, "created" : "2019-07-14T13:08:10.374Z", @@ -65,7 +65,7 @@ "error" : null, "user" : 2, "succeeded" : true, - "last_suceeded" : "2019-07-20T11:28:15.691Z", + "last_run" : "2019-07-20T11:28:15.691Z", "name" : "BBC", "modified" : "2019-07-20T12:07:49.164Z", "timezone" : "UTC", @@ -85,7 +85,7 @@ "website_url" : null, "name" : "Ars Technica", "succeeded" : true, - "last_suceeded" : "2019-07-20T11:28:15.986Z", + "last_run" : "2019-07-20T11:28:15.986Z", "modified" : "2019-07-20T11:28:16.033Z", "user" : 2 }, @@ -102,7 +102,7 @@ "user" : 2, "name" : "The Guardian", "succeeded" : true, - "last_suceeded" : "2019-07-20T11:28:16.078Z", + "last_run" : "2019-07-20T11:28:16.078Z", "modified" : "2019-07-20T12:07:44.292Z", "created" : "2019-07-20T11:25:02.089Z", "website_url" : null, @@ -119,7 +119,7 @@ "website_url" : null, "created" : "2019-07-20T11:25:30.121Z", "user" : 2, - "last_suceeded" : "2019-07-20T11:28:15.860Z", + "last_run" : "2019-07-20T11:28:15.860Z", "succeeded" : true, "modified" : "2019-07-20T12:07:28.473Z", "name" : "Tweakers" @@ -139,7 +139,7 @@ "website_url" : null, "timezone" : "UTC", "user" : 2, - "last_suceeded" : "2019-07-20T11:28:16.034Z", + "last_run" : "2019-07-20T11:28:16.034Z", "succeeded" : true, "modified" : "2019-07-20T12:07:21.704Z", "name" : "The Verge" diff --git a/src/newsreader/news/collection/admin.py b/src/newsreader/news/collection/admin.py index c5a7c5c..ece5c23 100644 --- a/src/newsreader/news/collection/admin.py +++ b/src/newsreader/news/collection/admin.py @@ -6,14 +6,7 @@ from newsreader.news.collection.models import CollectionRule class CollectionRuleAdmin(admin.ModelAdmin): fields = ("url", "name", "timezone", "category", "favicon", "user") - list_display = ( - "name", - "type_display", - "category", - "url", - "last_suceeded", - "succeeded", - ) + list_display = ("name", "type_display", "category", "url", "last_run", "succeeded") list_filter = ("user",) def save_model(self, request, obj, form, change): diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index 700fae5..1377078 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -108,6 +108,15 @@ class Collector: abstract = True +class Scheduler: + """ + Schedules rules according to certain ratelimitting + """ + + def get_scheduled_rules(self): + raise NotImplementedError + + class PostBuilder(Builder): rule_type = None diff --git a/src/newsreader/news/collection/choices.py b/src/newsreader/news/collection/choices.py index 3fd9bef..612079c 100644 --- a/src/newsreader/news/collection/choices.py +++ b/src/newsreader/news/collection/choices.py @@ -5,7 +5,7 @@ from django.utils.translation import gettext as _ class RuleTypeChoices(TextChoices): feed = "feed", _("Feed") subreddit = "subreddit", _("Subreddit") - twitter = "twitter", _("Twitter") + twitter_timeline = "twitter_timeline", _("Twitter timeline") class TwitterPostTypeChoices(TextChoices): diff --git a/src/newsreader/news/collection/feed.py b/src/newsreader/news/collection/feed.py index 22f8dc7..ae6cd42 100644 --- a/src/newsreader/news/collection/feed.py +++ b/src/newsreader/news/collection/feed.py @@ -118,7 +118,6 @@ class FeedClient(PostClient): stream.rule.error = None stream.rule.succeeded = True - stream.rule.last_suceeded = timezone.now() yield payload except (StreamNotFoundException, StreamTimeOutException) as e: @@ -134,6 +133,7 @@ class FeedClient(PostClient): continue finally: + stream.rule.last_run = timezone.now() stream.rule.save() diff --git a/src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py b/src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py new file mode 100644 index 0000000..2f08f6e --- /dev/null +++ b/src/newsreader/news/collection/migrations/0010_auto_20200913_2101.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-09-13 19:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("collection", "0009_auto_20200807_2030")] + + operations = [ + migrations.AlterField( + model_name="collectionrule", + name="type", + field=models.CharField( + choices=[ + ("feed", "Feed"), + ("subreddit", "Subreddit"), + ("twitter_timeline", "Twitter timeline"), + ], + default="feed", + max_length=20, + ), + ) + ] diff --git a/src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py b/src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py new file mode 100644 index 0000000..308c654 --- /dev/null +++ b/src/newsreader/news/collection/migrations/0011_auto_20200913_2157.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-09-13 19:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("collection", "0010_auto_20200913_2101")] + + operations = [ + migrations.RenameField( + model_name="collectionrule", old_name="last_suceeded", new_name="last_run" + ) + ] diff --git a/src/newsreader/news/collection/models.py b/src/newsreader/news/collection/models.py index 5bada6a..0cbfa22 100644 --- a/src/newsreader/news/collection/models.py +++ b/src/newsreader/news/collection/models.py @@ -41,9 +41,8 @@ class CollectionRule(TimeStampedModel): on_delete=models.SET_NULL, ) - last_suceeded = models.DateTimeField(blank=True, null=True) + last_run = models.DateTimeField(blank=True, null=True) succeeded = models.BooleanField(default=False) - error = models.CharField(max_length=1024, blank=True, null=True) enabled = models.BooleanField( diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index b905f24..0122bb1 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -20,6 +20,7 @@ from newsreader.news.collection.base import ( PostClient, PostCollector, PostStream, + Scheduler, ) from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.constants import ( @@ -239,6 +240,7 @@ class RedditClient(PostClient): futures = {executor.submit(stream.read): stream for stream in batch} if rate_limitted: + logger.warning("Aborting requests, ratelimit hit") break for future in as_completed(futures): @@ -249,7 +251,6 @@ class RedditClient(PostClient): stream.rule.error = None stream.rule.succeeded = True - stream.rule.last_suceeded = timezone.now() yield response_data except StreamDeniedException as e: @@ -281,6 +282,7 @@ class RedditClient(PostClient): continue finally: + stream.rule.last_run = timezone.now() stream.rule.save() @@ -289,7 +291,7 @@ class RedditCollector(PostCollector): client = RedditClient -class RedditScheduler: +class RedditScheduler(Scheduler): max_amount = RATE_LIMIT max_user_amount = RATE_LIMIT / 4 @@ -300,7 +302,7 @@ class RedditScheduler: user__reddit_access_token__isnull=False, user__reddit_refresh_token__isnull=False, enabled=True, - ).order_by("last_suceeded")[:200] + ).order_by("last_run")[:200] else: self.subreddits = subreddits diff --git a/src/newsreader/news/collection/tests/factories.py b/src/newsreader/news/collection/tests/factories.py index 761844b..81fa8ac 100644 --- a/src/newsreader/news/collection/tests/factories.py +++ b/src/newsreader/news/collection/tests/factories.py @@ -30,6 +30,6 @@ class SubredditFactory(CollectionRuleFactory): website_url = REDDIT_URL -class TwitterProfileFactory(CollectionRuleFactory): - type = RuleTypeChoices.twitter +class TwitterTimeLineFactory(CollectionRuleFactory): + type = RuleTypeChoices.twitter_timeline screen_name = factory.Faker("user_name") diff --git a/src/newsreader/news/collection/tests/feed/collector/tests.py b/src/newsreader/news/collection/tests/feed/collector/tests.py index 048d618..fea10bd 100644 --- a/src/newsreader/news/collection/tests/feed/collector/tests.py +++ b/src/newsreader/news/collection/tests/feed/collector/tests.py @@ -26,6 +26,7 @@ from newsreader.news.core.tests.factories import FeedPostFactory from .mocks import duplicate_mock, empty_mock, multiple_mock, multiple_update_mock +@freeze_time("2019-10-30 12:30:00") class FeedCollectorTestCase(TestCase): def setUp(self): self.maxDiff = None @@ -39,7 +40,6 @@ class FeedCollectorTestCase(TestCase): def tearDown(self): patch.stopall() - @freeze_time("2019-10-30 12:30:00") def test_simple_batch(self): self.mocked_parse.return_value = multiple_mock rule = FeedFactory() @@ -51,10 +51,9 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 3) self.assertEquals(rule.succeeded, True) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) self.assertEquals(rule.error, None) - @freeze_time("2019-10-30 12:30:00") def test_emtpy_batch(self): self.mocked_fetch.return_value = Mock() self.mocked_parse.return_value = empty_mock @@ -68,7 +67,7 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, True) self.assertEquals(rule.error, None) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) def test_not_found(self): self.mocked_fetch.side_effect = StreamNotFoundException @@ -85,10 +84,8 @@ class FeedCollectorTestCase(TestCase): def test_denied(self): self.mocked_fetch.side_effect = StreamDeniedException - last_suceeded = timezone.make_aware( - datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) - ) - rule = FeedFactory(last_suceeded=last_suceeded) + old_run = timezone.make_aware(datetime(2019, 10, 30, 12, 30)) + rule = FeedFactory(last_run=old_run) collector = FeedCollector() collector.collect() @@ -98,14 +95,12 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, False) self.assertEquals(rule.error, "Stream does not have sufficient permissions") - self.assertEquals(rule.last_suceeded, last_suceeded) + self.assertEquals(rule.last_run, timezone.now()) def test_forbidden(self): self.mocked_fetch.side_effect = StreamForbiddenException - last_suceeded = timezone.make_aware( - datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) - ) - rule = FeedFactory(last_suceeded=last_suceeded) + old_run = pytz.utc.localize(datetime(2019, 10, 30, 12, 30)) + rule = FeedFactory(last_run=old_run) collector = FeedCollector() collector.collect() @@ -115,14 +110,14 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, False) self.assertEquals(rule.error, "Stream forbidden") - self.assertEquals(rule.last_suceeded, last_suceeded) + self.assertEquals(rule.last_run, timezone.now()) def test_timed_out(self): self.mocked_fetch.side_effect = StreamTimeOutException - last_suceeded = timezone.make_aware( + last_run = timezone.make_aware( datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) ) - rule = FeedFactory(last_suceeded=last_suceeded) + rule = FeedFactory(last_run=last_run) collector = FeedCollector() collector.collect() @@ -132,9 +127,10 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 0) self.assertEquals(rule.succeeded, False) self.assertEquals(rule.error, "Stream timed out") - self.assertEquals(rule.last_suceeded, last_suceeded) + self.assertEquals( + rule.last_run, pytz.utc.localize(datetime(2019, 10, 30, 12, 30)) + ) - @freeze_time("2019-10-30 12:30:00") def test_duplicates(self): self.mocked_parse.return_value = duplicate_mock rule = FeedFactory() @@ -186,10 +182,9 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 3) self.assertEquals(rule.succeeded, True) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) self.assertEquals(rule.error, None) - @freeze_time("2019-02-22 12:30:00") def test_items_with_identifiers_get_updated(self): self.mocked_parse.return_value = multiple_update_mock rule = FeedFactory() @@ -231,7 +226,7 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 3) self.assertEquals(rule.succeeded, True) - self.assertEquals(rule.last_suceeded, timezone.now()) + self.assertEquals(rule.last_run, timezone.now()) self.assertEquals(rule.error, None) self.assertEquals( @@ -246,9 +241,12 @@ class FeedCollectorTestCase(TestCase): third_post.title, "Birmingham head teacher threatened over LGBT lessons" ) - @freeze_time("2019-02-22 12:30:00") def test_disabled_rules(self): - rules = (FeedFactory(enabled=False), FeedFactory(enabled=True)) + old_run = pytz.utc.localize(datetime(2019, 10, 28, 15)) + rules = ( + FeedFactory(enabled=False, last_run=old_run), + FeedFactory(enabled=True, last_run=old_run), + ) self.mocked_parse.return_value = multiple_mock @@ -260,8 +258,8 @@ class FeedCollectorTestCase(TestCase): self.assertEquals(Post.objects.count(), 3) self.assertEquals(rules[1].succeeded, True) - self.assertEquals(rules[1].last_suceeded, timezone.now()) + self.assertEquals(rules[1].last_run, timezone.now()) self.assertEquals(rules[1].error, None) - self.assertEquals(rules[0].last_suceeded, None) + self.assertEquals(rules[0].last_run, old_run) self.assertEquals(rules[0].succeeded, False) diff --git a/src/newsreader/news/collection/tests/reddit/collector/tests.py b/src/newsreader/news/collection/tests/reddit/collector/tests.py index 1fd18b0..fa2f5d4 100644 --- a/src/newsreader/news/collection/tests/reddit/collector/tests.py +++ b/src/newsreader/news/collection/tests/reddit/collector/tests.py @@ -74,7 +74,7 @@ class RedditCollectorTestCase(TestCase): for subreddit in rules: with self.subTest(subreddit=subreddit): self.assertEquals(subreddit.succeeded, True) - self.assertEquals(subreddit.last_suceeded, timezone.now()) + self.assertEquals(subreddit.last_run, timezone.now()) self.assertEquals(subreddit.error, None) post = Post.objects.get( @@ -133,7 +133,7 @@ class RedditCollectorTestCase(TestCase): for subreddit in rules: with self.subTest(subreddit=subreddit): self.assertEquals(subreddit.succeeded, True) - self.assertEquals(subreddit.last_suceeded, timezone.now()) + self.assertEquals(subreddit.last_run, timezone.now()) self.assertEquals(subreddit.error, None) def test_not_found(self): diff --git a/src/newsreader/news/collection/tests/reddit/test_scheduler.py b/src/newsreader/news/collection/tests/reddit/test_scheduler.py index cd062b6..0f04d53 100644 --- a/src/newsreader/news/collection/tests/reddit/test_scheduler.py +++ b/src/newsreader/news/collection/tests/reddit/test_scheduler.py @@ -25,19 +25,19 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory( user=user_1, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=4), + last_run=timezone.now() - timedelta(days=4), enabled=True, ), CollectionRuleFactory( user=user_1, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=3), + last_run=timezone.now() - timedelta(days=3), enabled=True, ), CollectionRuleFactory( user=user_1, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=2), + last_run=timezone.now() - timedelta(days=2), enabled=True, ), ] @@ -46,19 +46,19 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory( user=user_2, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=4), + last_run=timezone.now() - timedelta(days=4), enabled=True, ), CollectionRuleFactory( user=user_2, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=3), + last_run=timezone.now() - timedelta(days=3), enabled=True, ), CollectionRuleFactory( user=user_2, type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(days=2), + last_run=timezone.now() - timedelta(days=2), enabled=True, ), ] @@ -87,7 +87,7 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory.create_batch( name=f"rule-{index}", type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(seconds=index), + last_run=timezone.now() - timedelta(seconds=index), enabled=True, user=user, size=15, @@ -121,7 +121,7 @@ class RedditSchedulerTestCase(TestCase): CollectionRuleFactory( name=f"rule-{index}", type=RuleTypeChoices.subreddit, - last_suceeded=timezone.now() - timedelta(seconds=index), + last_run=timezone.now() - timedelta(seconds=index), enabled=True, user=user, ) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index cc43c3c..2527fb9 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -8,7 +8,7 @@ import pytz from ftfy import fix_text -from newsreader.news.collection.tests.factories import TwitterProfileFactory +from newsreader.news.collection.tests.factories import TwitterTimeLineFactory from newsreader.news.collection.tests.twitter.builder.mocks import ( gif_mock, image_mock, @@ -31,7 +31,7 @@ class TwitterBuilderTestCase(TestCase): def test_simple_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(simple_mock, mock_stream) as builder: @@ -91,7 +91,7 @@ class TwitterBuilderTestCase(TestCase): def test_images_in_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(image_mock, mock_stream) as builder: @@ -134,7 +134,7 @@ class TwitterBuilderTestCase(TestCase): def test_videos_in_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(video_mock, mock_stream) as builder: @@ -190,7 +190,7 @@ class TwitterBuilderTestCase(TestCase): def test_video_without_bitrate(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(video_without_bitrate_mock, mock_stream) as builder: @@ -212,7 +212,7 @@ class TwitterBuilderTestCase(TestCase): def test_GIFs_in_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(gif_mock, mock_stream) as builder: @@ -241,7 +241,7 @@ class TwitterBuilderTestCase(TestCase): def test_retweet_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(retweet_mock, mock_stream) as builder: @@ -278,7 +278,7 @@ class TwitterBuilderTestCase(TestCase): def test_quoted_post(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(quoted_mock, mock_stream) as builder: @@ -312,7 +312,7 @@ class TwitterBuilderTestCase(TestCase): def test_empty_data(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder([], mock_stream) as builder: @@ -324,7 +324,7 @@ class TwitterBuilderTestCase(TestCase): def test_html_sanitizing(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(unsanitized_mock, mock_stream) as builder: @@ -364,7 +364,7 @@ class TwitterBuilderTestCase(TestCase): def test_urlize_on_urls(self): builder = TwitterBuilder - profile = TwitterProfileFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(simple_mock, mock_stream) as builder: diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 19fb812..c2d693f 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -1,6 +1,7 @@ import logging from datetime import datetime +from json import JSONDecodeError from django.utils.html import format_html, urlize @@ -13,9 +14,16 @@ from newsreader.news.collection.base import ( PostClient, PostCollector, PostStream, + Scheduler, ) from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices -from newsreader.news.collection.utils import truncate_text +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamParseException, + StreamTooManyException, +) +from newsreader.news.collection.utils import fetch, truncate_text from newsreader.news.core.models import Post @@ -26,7 +34,7 @@ TWITTER_API_URL = "https://api.twitter.com/1.1" class TwitterBuilder(PostBuilder): - rule_type = RuleTypeChoices.twitter + rule_type = RuleTypeChoices.twitter_timeline def build(self): results = {} @@ -147,5 +155,49 @@ class TwitterCollector(PostCollector): pass -class TwitterScheduler: - pass +# see https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits +class TwitterTimeLineScheduler(Scheduler): + def __init__(self, user, timelines=[]): + self.user = user + + if not timelines: + self.timelines = user.rules.enabled( + type=RuleTypeChoices.twitter_timeline + ).order_by("last_run")[:200] + else: + self.timelines = timelines + + def get_scheduled_rules(self): + if ( + not self.user.twitter_oauth_token + or not self.user.twitter_oauth_token_secret + ): + return [] + + max_amount = self.get_current_ratelimit() + return self.timelines[:max_amount] if max_amount else [] + + def get_current_ratelimit(self): + endpoint = ( + "application/rate_limit_status.json?resources=help,users,search,statuses" + ) + + # TODO add appropriate authentication (OAuth 1.0a) headers + try: + response = fetch(f"{TWITTER_API_URL}/{endpoint}") + except StreamException: + logger.exception(f"Unable to retrieve current ratelimit for {self.user.pk}") + return + + try: + payload = response.json() + except JSONDecodeError: + logger.exception(f"Unable to parse ratelimit request for {self.user.pk}") + return + + if not "resources" in payload or not "statuses" in payload["resources"]: + return [] + + statuses = payload["resources"]["statuses"] + + return statuses.get("/statuses/user_timeline", 0) From 21fc5ef8e7f4148e1cbb83b351bc3d3239467427 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 14 Sep 2020 20:09:06 +0200 Subject: [PATCH 21/69] Add twitter settings example --- src/newsreader/conf/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/newsreader/conf/base.py b/src/newsreader/conf/base.py index 7b8c0b6..94ce2be 100644 --- a/src/newsreader/conf/base.py +++ b/src/newsreader/conf/base.py @@ -203,6 +203,11 @@ REDDIT_CLIENT_ID = "CLIENT_ID" REDDIT_CLIENT_SECRET = "CLIENT_SECRET" REDDIT_REDIRECT_URL = "http://127.0.0.1:8000/accounts/settings/reddit/callback/" +# Twitter integration +TWITTER_CONSUMER_ID = "CONSUMER_ID" +TWITTER_CONSUMER_SECRET = "CONSUMER_SECRET" +TWITTER_REDIRECT_URL = "http://127.0.0.1:8000/accounts/settings/twitter/callback/" + # Third party settings AXES_HANDLER = "axes.handlers.cache.AxesCacheHandler" AXES_CACHE = "axes" From 87378a35339b82a6d01233f57bb412a6763233ee Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 14 Sep 2020 20:42:03 +0200 Subject: [PATCH 22/69] Split account views into seperate files --- src/newsreader/accounts/views.py | 210 ------------------ src/newsreader/accounts/views/__init__.py | 21 ++ src/newsreader/accounts/views/auth.py | 11 + src/newsreader/accounts/views/integrations.py | 75 +++++++ src/newsreader/accounts/views/password.py | 37 +++ src/newsreader/accounts/views/registration.py | 59 +++++ src/newsreader/accounts/views/settings.py | 50 +++++ 7 files changed, 253 insertions(+), 210 deletions(-) delete mode 100644 src/newsreader/accounts/views.py create mode 100644 src/newsreader/accounts/views/__init__.py create mode 100644 src/newsreader/accounts/views/auth.py create mode 100644 src/newsreader/accounts/views/integrations.py create mode 100644 src/newsreader/accounts/views/password.py create mode 100644 src/newsreader/accounts/views/registration.py create mode 100644 src/newsreader/accounts/views/settings.py diff --git a/src/newsreader/accounts/views.py b/src/newsreader/accounts/views.py deleted file mode 100644 index 4f982a9..0000000 --- a/src/newsreader/accounts/views.py +++ /dev/null @@ -1,210 +0,0 @@ -from django.contrib import messages -from django.contrib.auth import views as django_views -from django.core.cache import cache -from django.shortcuts import render -from django.urls import reverse_lazy -from django.utils.translation import gettext as _ -from django.views.generic import RedirectView, TemplateView -from django.views.generic.edit import FormView, ModelFormMixin - -from registration.backends.default import views as registration_views - -from newsreader.accounts.forms import UserSettingsForm -from newsreader.accounts.models import User -from newsreader.news.collection.exceptions import StreamException -from newsreader.news.collection.reddit import ( - get_reddit_access_token, - get_reddit_authorization_url, -) -from newsreader.news.collection.tasks import RedditTokenTask - - -class LoginView(django_views.LoginView): - template_name = "accounts/views/login.html" - success_url = reverse_lazy("index") - - -class LogoutView(django_views.LogoutView): - next_page = reverse_lazy("accounts:login") - - -# RegistrationView shows a registration form and sends the email -# RegistrationCompleteView shows after filling in the registration form -# ActivationView is send within the activation email and activates the account -# ActivationCompleteView shows the success screen when activation was succesful -# ActivationResendView can be used when activation links are expired -# RegistrationClosedView shows when registration is disabled -class RegistrationView(registration_views.RegistrationView): - disallowed_url = reverse_lazy("accounts:register-closed") - template_name = "registration/registration_form.html" - success_url = reverse_lazy("accounts:register-complete") - - -class RegistrationCompleteView(TemplateView): - template_name = "registration/registration_complete.html" - - -class RegistrationClosedView(TemplateView): - template_name = "registration/registration_closed.html" - - -# Redirects or renders failed activation template -class ActivationView(registration_views.ActivationView): - template_name = "registration/activation_failure.html" - - def get_success_url(self, user): - return ("accounts:activate-complete", (), {}) - - -class ActivationCompleteView(TemplateView): - template_name = "registration/activation_complete.html" - - -# Renders activation form resend or resend_activation_complete -class ActivationResendView(registration_views.ResendActivationView): - template_name = "registration/activation_resend_form.html" - - def render_form_submitted_template(self, form): - """ - Renders resend activation complete template with the submitted email. - - """ - email = form.cleaned_data["email"] - context = {"email": email} - - return render( - self.request, "registration/activation_resend_complete.html", context - ) - - -# PasswordResetView sends the mail -# PasswordResetDoneView shows a success message for the above -# PasswordResetConfirmView checks the link the user clicked and -# prompts for a new password -# PasswordResetCompleteView shows a success message for the above -class PasswordResetView(django_views.PasswordResetView): - template_name = "password-reset/password-reset.html" - subject_template_name = "password-reset/password-reset-subject.txt" - email_template_name = "password-reset/password-reset-email.html" - success_url = reverse_lazy("accounts:password-reset-done") - - -class PasswordResetDoneView(django_views.PasswordResetDoneView): - template_name = "password-reset/password-reset-done.html" - - -class PasswordResetConfirmView(django_views.PasswordResetConfirmView): - template_name = "password-reset/password-reset-confirm.html" - success_url = reverse_lazy("accounts:password-reset-complete") - - -class PasswordResetCompleteView(django_views.PasswordResetCompleteView): - template_name = "password-reset/password-reset-complete.html" - - -class PasswordChangeView(django_views.PasswordChangeView): - template_name = "accounts/views/password-change.html" - success_url = reverse_lazy("accounts:settings") - - -class SettingsView(ModelFormMixin, FormView): - template_name = "accounts/views/settings.html" - success_url = reverse_lazy("accounts:settings") - form_class = UserSettingsForm - model = User - - def get(self, request, *args, **kwargs): - self.object = self.get_object() - return super().get(request, *args, **kwargs) - - def get_object(self, **kwargs): - return self.request.user - - def get_context_data(self, **kwargs): - user = self.request.user - - reddit_authorization_url = None - reddit_refresh_url = None - reddit_task_active = cache.get(f"{user.email}-reddit-refresh") - - if ( - user.reddit_refresh_token - and not user.reddit_access_token - and not reddit_task_active - ): - reddit_refresh_url = reverse_lazy("accounts:reddit-refresh") - - if not user.reddit_refresh_token: - reddit_authorization_url = get_reddit_authorization_url(user) - - return { - **super().get_context_data(**kwargs), - "reddit_authorization_url": reddit_authorization_url, - "reddit_refresh_url": reddit_refresh_url, - } - - def get_form_kwargs(self): - return {**super().get_form_kwargs(), "instance": self.request.user} - - -class RedditTemplateView(TemplateView): - template_name = "accounts/views/reddit.html" - - def get(self, request, *args, **kwargs): - context = self.get_context_data(**kwargs) - - error = request.GET.get("error", None) - state = request.GET.get("state", None) - code = request.GET.get("code", None) - - if error: - return self.render_to_response({**context, "error": error}) - - if not code or not state: - return self.render_to_response(context) - - cached_state = cache.get(f"{request.user.email}-reddit-auth") - - if state != cached_state: - return self.render_to_response( - { - **context, - "error": "The saved state for Reddit authorization did not match", - } - ) - - try: - access_token, refresh_token = get_reddit_access_token(code, request.user) - - return self.render_to_response( - { - **context, - "access_token": access_token, - "refresh_token": refresh_token, - } - ) - except StreamException as e: - return self.render_to_response({**context, "error": str(e)}) - except KeyError: - return self.render_to_response( - {**context, "error": "Access and refresh token not found in response"} - ) - - -class RedditTokenRedirectView(RedirectView): - url = reverse_lazy("accounts:settings") - - def get(self, request, *args, **kwargs): - response = super().get(request, *args, **kwargs) - - user = request.user - task_active = cache.get(f"{user.email}-reddit-refresh") - - if not task_active: - RedditTokenTask.delay(user.pk) - messages.success(request, _("Access token is being retrieved")) - cache.set(f"{user.email}-reddit-refresh", 1, 300) - return response - - messages.error(request, _("Unable to retrieve token")) - return response diff --git a/src/newsreader/accounts/views/__init__.py b/src/newsreader/accounts/views/__init__.py new file mode 100644 index 0000000..ee44f5b --- /dev/null +++ b/src/newsreader/accounts/views/__init__.py @@ -0,0 +1,21 @@ +from newsreader.accounts.views.auth import LoginView, LogoutView +from newsreader.accounts.views.integrations import ( + RedditTemplateView, + RedditTokenRedirectView, +) +from newsreader.accounts.views.password import ( + PasswordChangeView, + PasswordResetCompleteView, + PasswordResetConfirmView, + PasswordResetDoneView, + PasswordResetView, +) +from newsreader.accounts.views.registration import ( + ActivationCompleteView, + ActivationResendView, + ActivationView, + RegistrationClosedView, + RegistrationCompleteView, + RegistrationView, +) +from newsreader.accounts.views.settings import SettingsView diff --git a/src/newsreader/accounts/views/auth.py b/src/newsreader/accounts/views/auth.py new file mode 100644 index 0000000..0663768 --- /dev/null +++ b/src/newsreader/accounts/views/auth.py @@ -0,0 +1,11 @@ +from django.contrib.auth import views as django_views +from django.urls import reverse_lazy + + +class LoginView(django_views.LoginView): + template_name = "accounts/views/login.html" + success_url = reverse_lazy("index") + + +class LogoutView(django_views.LogoutView): + next_page = reverse_lazy("accounts:login") diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py new file mode 100644 index 0000000..62edc1c --- /dev/null +++ b/src/newsreader/accounts/views/integrations.py @@ -0,0 +1,75 @@ +from django.contrib import messages +from django.core.cache import cache +from django.urls import reverse_lazy +from django.utils.translation import gettext as _ +from django.views.generic import RedirectView, TemplateView + +from newsreader.news.collection.exceptions import StreamException +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) +from newsreader.news.collection.tasks import RedditTokenTask + + +class RedditTemplateView(TemplateView): + template_name = "accounts/views/reddit.html" + + def get(self, request, *args, **kwargs): + context = self.get_context_data(**kwargs) + + error = request.GET.get("error", None) + state = request.GET.get("state", None) + code = request.GET.get("code", None) + + if error: + return self.render_to_response({**context, "error": error}) + + if not code or not state: + return self.render_to_response(context) + + cached_state = cache.get(f"{request.user.email}-reddit-auth") + + if state != cached_state: + return self.render_to_response( + { + **context, + "error": "The saved state for Reddit authorization did not match", + } + ) + + try: + access_token, refresh_token = get_reddit_access_token(code, request.user) + + return self.render_to_response( + { + **context, + "access_token": access_token, + "refresh_token": refresh_token, + } + ) + except StreamException as e: + return self.render_to_response({**context, "error": str(e)}) + except KeyError: + return self.render_to_response( + {**context, "error": "Access and refresh token not found in response"} + ) + + +class RedditTokenRedirectView(RedirectView): + url = reverse_lazy("accounts:settings") + + def get(self, request, *args, **kwargs): + response = super().get(request, *args, **kwargs) + + user = request.user + task_active = cache.get(f"{user.email}-reddit-refresh") + + if not task_active: + RedditTokenTask.delay(user.pk) + messages.success(request, _("Access token is being retrieved")) + cache.set(f"{user.email}-reddit-refresh", 1, 300) + return response + + messages.error(request, _("Unable to retrieve token")) + return response diff --git a/src/newsreader/accounts/views/password.py b/src/newsreader/accounts/views/password.py new file mode 100644 index 0000000..e9e0aa3 --- /dev/null +++ b/src/newsreader/accounts/views/password.py @@ -0,0 +1,37 @@ +from django.contrib.auth import views as django_views +from django.urls import reverse_lazy + +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) + + +# PasswordResetView sends the mail +# PasswordResetDoneView shows a success message for the above +# PasswordResetConfirmView checks the link the user clicked and +# prompts for a new password +# PasswordResetCompleteView shows a success message for the above +class PasswordResetView(django_views.PasswordResetView): + template_name = "password-reset/password-reset.html" + subject_template_name = "password-reset/password-reset-subject.txt" + email_template_name = "password-reset/password-reset-email.html" + success_url = reverse_lazy("accounts:password-reset-done") + + +class PasswordResetDoneView(django_views.PasswordResetDoneView): + template_name = "password-reset/password-reset-done.html" + + +class PasswordResetConfirmView(django_views.PasswordResetConfirmView): + template_name = "password-reset/password-reset-confirm.html" + success_url = reverse_lazy("accounts:password-reset-complete") + + +class PasswordResetCompleteView(django_views.PasswordResetCompleteView): + template_name = "password-reset/password-reset-complete.html" + + +class PasswordChangeView(django_views.PasswordChangeView): + template_name = "accounts/views/password-change.html" + success_url = reverse_lazy("accounts:settings") diff --git a/src/newsreader/accounts/views/registration.py b/src/newsreader/accounts/views/registration.py new file mode 100644 index 0000000..597aa9a --- /dev/null +++ b/src/newsreader/accounts/views/registration.py @@ -0,0 +1,59 @@ +from django.shortcuts import render +from django.urls import reverse_lazy +from django.views.generic import TemplateView + +from registration.backends.default import views as registration_views + +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) + + +# RegistrationView shows a registration form and sends the email +# RegistrationCompleteView shows after filling in the registration form +# ActivationView is send within the activation email and activates the account +# ActivationCompleteView shows the success screen when activation was succesful +# ActivationResendView can be used when activation links are expired +# RegistrationClosedView shows when registration is disabled +class RegistrationView(registration_views.RegistrationView): + disallowed_url = reverse_lazy("accounts:register-closed") + template_name = "registration/registration_form.html" + success_url = reverse_lazy("accounts:register-complete") + + +class RegistrationCompleteView(TemplateView): + template_name = "registration/registration_complete.html" + + +class RegistrationClosedView(TemplateView): + template_name = "registration/registration_closed.html" + + +# Redirects or renders failed activation template +class ActivationView(registration_views.ActivationView): + template_name = "registration/activation_failure.html" + + def get_success_url(self, user): + return ("accounts:activate-complete", (), {}) + + +class ActivationCompleteView(TemplateView): + template_name = "registration/activation_complete.html" + + +# Renders activation form resend or resend_activation_complete +class ActivationResendView(registration_views.ResendActivationView): + template_name = "registration/activation_resend_form.html" + + def render_form_submitted_template(self, form): + """ + Renders resend activation complete template with the submitted email. + + """ + email = form.cleaned_data["email"] + context = {"email": email} + + return render( + self.request, "registration/activation_resend_complete.html", context + ) diff --git a/src/newsreader/accounts/views/settings.py b/src/newsreader/accounts/views/settings.py new file mode 100644 index 0000000..1fdff3b --- /dev/null +++ b/src/newsreader/accounts/views/settings.py @@ -0,0 +1,50 @@ +from django.core.cache import cache +from django.urls import reverse_lazy +from django.views.generic.edit import FormView, ModelFormMixin + +from newsreader.accounts.forms import UserSettingsForm +from newsreader.accounts.models import User +from newsreader.news.collection.reddit import ( + get_reddit_access_token, + get_reddit_authorization_url, +) + + +class SettingsView(ModelFormMixin, FormView): + template_name = "accounts/views/settings.html" + success_url = reverse_lazy("accounts:settings") + form_class = UserSettingsForm + model = User + + def get(self, request, *args, **kwargs): + self.object = self.get_object() + return super().get(request, *args, **kwargs) + + def get_object(self, **kwargs): + return self.request.user + + def get_context_data(self, **kwargs): + user = self.request.user + + reddit_authorization_url = None + reddit_refresh_url = None + reddit_task_active = cache.get(f"{user.email}-reddit-refresh") + + if ( + user.reddit_refresh_token + and not user.reddit_access_token + and not reddit_task_active + ): + reddit_refresh_url = reverse_lazy("accounts:reddit-refresh") + + if not user.reddit_refresh_token: + reddit_authorization_url = get_reddit_authorization_url(user) + + return { + **super().get_context_data(**kwargs), + "reddit_authorization_url": reddit_authorization_url, + "reddit_refresh_url": reddit_refresh_url, + } + + def get_form_kwargs(self): + return {**super().get_form_kwargs(), "instance": self.request.user} From 19270513d51463955db41f3b1140a077541679af Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 14 Sep 2020 20:53:22 +0200 Subject: [PATCH 23/69] Update form components --- .../accounts/templates/accounts/components/settings-form.html | 4 ---- src/newsreader/templates/components/form/title.html | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/newsreader/accounts/templates/accounts/components/settings-form.html b/src/newsreader/accounts/templates/accounts/components/settings-form.html index 7942354..caed407 100644 --- a/src/newsreader/accounts/templates/accounts/components/settings-form.html +++ b/src/newsreader/accounts/templates/accounts/components/settings-form.html @@ -3,10 +3,6 @@ {% block actions %}
-
- {% include "components/form/cancel-button.html" %} -
-
{% trans "Change password" %} diff --git a/src/newsreader/templates/components/form/title.html b/src/newsreader/templates/components/form/title.html index 3adcb75..8d6ded7 100644 --- a/src/newsreader/templates/components/form/title.html +++ b/src/newsreader/templates/components/form/title.html @@ -1,3 +1,3 @@ -
+

{{ title }}

-
+ From df6fd067c30f577b7951878290f3094ab814623b Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 14 Sep 2020 23:25:11 +0200 Subject: [PATCH 24/69] Add integrations view --- .../accounts/components/settings-form.html | 15 ++----- .../accounts/views/integrations.html | 43 +++++++++++++++++++ src/newsreader/accounts/urls.py | 18 ++++++-- src/newsreader/accounts/views/__init__.py | 1 + src/newsreader/accounts/views/integrations.py | 32 ++++++++++++++ src/newsreader/accounts/views/settings.py | 24 ----------- .../scss/components/header/_header.scss | 3 ++ .../scss/components/header/index.scss | 1 + src/newsreader/scss/components/index.scss | 3 ++ .../integrations/_integrations.scss | 12 ++++++ .../scss/components/integrations/index.scss | 1 + .../scss/elements/button/_button.scss | 18 +++++++- src/newsreader/scss/pages/index.scss | 1 + .../scss/pages/integrations/index.scss | 5 +++ src/newsreader/scss/partials/_colors.scss | 1 + .../templates/components/form/form.html | 2 +- .../templates/components/form/title.html | 3 -- .../templates/components/header/header.html | 3 ++ 18 files changed, 141 insertions(+), 45 deletions(-) create mode 100644 src/newsreader/accounts/templates/accounts/views/integrations.html create mode 100644 src/newsreader/scss/components/header/_header.scss create mode 100644 src/newsreader/scss/components/header/index.scss create mode 100644 src/newsreader/scss/components/integrations/_integrations.scss create mode 100644 src/newsreader/scss/components/integrations/index.scss create mode 100644 src/newsreader/scss/pages/integrations/index.scss delete mode 100644 src/newsreader/templates/components/form/title.html create mode 100644 src/newsreader/templates/components/header/header.html diff --git a/src/newsreader/accounts/templates/accounts/components/settings-form.html b/src/newsreader/accounts/templates/accounts/components/settings-form.html index caed407..51d4450 100644 --- a/src/newsreader/accounts/templates/accounts/components/settings-form.html +++ b/src/newsreader/accounts/templates/accounts/components/settings-form.html @@ -7,20 +7,11 @@ {% trans "Change password" %} + + {% trans "Third party integrations" %} + {% include "components/form/confirm-button.html" %} - - {% if reddit_authorization_url %} - - {% trans "Authorize Reddit account" %} - - {% endif %} - - {% if reddit_refresh_url %} - - {% trans "Refresh Reddit access token" %} - - {% endif %}
{% endblock actions %} diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html new file mode 100644 index 0000000..3ef2134 --- /dev/null +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -0,0 +1,43 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block content %} +
+
+ {% include "components/header/header.html" with title="Integrations" only %} + +
+

Reddit

+
+ + + + + +
+
+
+

Twitter

+
+ + + + + +
+
+
+
+{% endblock %} diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index 672cf6d..c250a0f 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -5,6 +5,7 @@ from newsreader.accounts.views import ( ActivationCompleteView, ActivationResendView, ActivationView, + IntegrationsView, LoginView, LogoutView, PasswordChangeView, @@ -22,8 +23,10 @@ from newsreader.accounts.views import ( urlpatterns = [ + # Auth path("login/", LoginView.as_view(), name="login"), path("logout/", LogoutView.as_view(), name="logout"), + # Register path("register/", RegistrationView.as_view(), name="register"), path( "register/complete/", @@ -41,6 +44,7 @@ urlpatterns = [ ActivationView.as_view(), name="activate", ), + # Password path("password-reset/", PasswordResetView.as_view(), name="password-reset"), path( "password-reset/done/", @@ -62,15 +66,23 @@ urlpatterns = [ login_required(PasswordChangeView.as_view()), name="password-change", ), - path("settings/", login_required(SettingsView.as_view()), name="settings"), + # Integrations + # TODO update reddit callback url in reddit app settings path( - "settings/reddit/callback/", + "settings/integrations/reddit/callback/", login_required(RedditTemplateView.as_view()), name="reddit-template", ), path( - "settings/reddit/refresh/", + "settings/integrations/reddit/refresh/", login_required(RedditTokenRedirectView.as_view()), name="reddit-refresh", ), + path( + "settings/integrations", + login_required(IntegrationsView.as_view()), + name="integrations", + ), + # Settings + path("settings/", login_required(SettingsView.as_view()), name="settings"), ] diff --git a/src/newsreader/accounts/views/__init__.py b/src/newsreader/accounts/views/__init__.py index ee44f5b..296cb28 100644 --- a/src/newsreader/accounts/views/__init__.py +++ b/src/newsreader/accounts/views/__init__.py @@ -1,5 +1,6 @@ from newsreader.accounts.views.auth import LoginView, LogoutView from newsreader.accounts.views.integrations import ( + IntegrationsView, RedditTemplateView, RedditTokenRedirectView, ) diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 62edc1c..a32c747 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -12,6 +12,38 @@ from newsreader.news.collection.reddit import ( from newsreader.news.collection.tasks import RedditTokenTask +class IntegrationsView(TemplateView): + template_name = "accounts/views/integrations.html" + + def get_context_data(self, **kwargs): + return { + **super().get_context_data(**kwargs), + **self.get_reddit_context(**kwargs), + } + + def get_reddit_context(self, **kwargs): + user = self.request.user + reddit_authorization_url = None + reddit_refresh_url = None + + reddit_task_active = cache.get(f"{user.email}-reddit-refresh") + + if ( + user.reddit_refresh_token + and not user.reddit_access_token + and not reddit_task_active + ): + reddit_refresh_url = reverse_lazy("accounts:reddit-refresh") + + if not user.reddit_refresh_token: + reddit_authorization_url = get_reddit_authorization_url(user) + + return { + "reddit_authorization_url": reddit_authorization_url, + "reddit_refresh_url": reddit_refresh_url, + } + + class RedditTemplateView(TemplateView): template_name = "accounts/views/reddit.html" diff --git a/src/newsreader/accounts/views/settings.py b/src/newsreader/accounts/views/settings.py index 1fdff3b..1603252 100644 --- a/src/newsreader/accounts/views/settings.py +++ b/src/newsreader/accounts/views/settings.py @@ -1,4 +1,3 @@ -from django.core.cache import cache from django.urls import reverse_lazy from django.views.generic.edit import FormView, ModelFormMixin @@ -23,28 +22,5 @@ class SettingsView(ModelFormMixin, FormView): def get_object(self, **kwargs): return self.request.user - def get_context_data(self, **kwargs): - user = self.request.user - - reddit_authorization_url = None - reddit_refresh_url = None - reddit_task_active = cache.get(f"{user.email}-reddit-refresh") - - if ( - user.reddit_refresh_token - and not user.reddit_access_token - and not reddit_task_active - ): - reddit_refresh_url = reverse_lazy("accounts:reddit-refresh") - - if not user.reddit_refresh_token: - reddit_authorization_url = get_reddit_authorization_url(user) - - return { - **super().get_context_data(**kwargs), - "reddit_authorization_url": reddit_authorization_url, - "reddit_refresh_url": reddit_refresh_url, - } - def get_form_kwargs(self): return {**super().get_form_kwargs(), "instance": self.request.user} diff --git a/src/newsreader/scss/components/header/_header.scss b/src/newsreader/scss/components/header/_header.scss new file mode 100644 index 0000000..ed96dc6 --- /dev/null +++ b/src/newsreader/scss/components/header/_header.scss @@ -0,0 +1,3 @@ +.header { + padding: 15px; +} diff --git a/src/newsreader/scss/components/header/index.scss b/src/newsreader/scss/components/header/index.scss new file mode 100644 index 0000000..5c23e3e --- /dev/null +++ b/src/newsreader/scss/components/header/index.scss @@ -0,0 +1 @@ +@import './header'; diff --git a/src/newsreader/scss/components/index.scss b/src/newsreader/scss/components/index.scss index cc9e717..b82a22d 100644 --- a/src/newsreader/scss/components/index.scss +++ b/src/newsreader/scss/components/index.scss @@ -8,6 +8,7 @@ @import './card/index'; @import './list/index'; +@import './header/index'; @import './messages/index'; @import './section/index'; @import './errorlist/index'; @@ -16,6 +17,8 @@ @import './sidebar/index'; @import './table/index'; +@import './integrations/index'; + @import './rules/index'; @import './category/index'; diff --git a/src/newsreader/scss/components/integrations/_integrations.scss b/src/newsreader/scss/components/integrations/_integrations.scss new file mode 100644 index 0000000..815184e --- /dev/null +++ b/src/newsreader/scss/components/integrations/_integrations.scss @@ -0,0 +1,12 @@ +.integrations { + display: flex; + flex-direction: column; + gap: 15px; + + padding: 15px; + + &__controls { + display: flex; + gap: 10px; + } +} diff --git a/src/newsreader/scss/components/integrations/index.scss b/src/newsreader/scss/components/integrations/index.scss new file mode 100644 index 0000000..7f9e759 --- /dev/null +++ b/src/newsreader/scss/components/integrations/index.scss @@ -0,0 +1 @@ +@import './integrations'; diff --git a/src/newsreader/scss/elements/button/_button.scss b/src/newsreader/scss/elements/button/_button.scss index a8eb3bc..7cd062a 100644 --- a/src/newsreader/scss/elements/button/_button.scss +++ b/src/newsreader/scss/elements/button/_button.scss @@ -44,10 +44,24 @@ &--reddit { color: $white !important; - background-color: lighten($reddit-orange, 5%); + background-color: $reddit-orange; &:hover { - background-color: $reddit-orange; + background-color: lighten($reddit-orange, 5%); } } + + &--twitter { + color: $white !important; + background-color: $twitter-blue; + + &:hover { + background-color: lighten($twitter-blue, 5%); + } + } + + &--disabled { + color: $font-color !important; + background-color: $gray !important; + } } diff --git a/src/newsreader/scss/pages/index.scss b/src/newsreader/scss/pages/index.scss index 44ca8a7..2ac0bb2 100644 --- a/src/newsreader/scss/pages/index.scss +++ b/src/newsreader/scss/pages/index.scss @@ -12,3 +12,4 @@ @import './rules/index'; @import './settings/index'; +@import './integrations/index'; diff --git a/src/newsreader/scss/pages/integrations/index.scss b/src/newsreader/scss/pages/integrations/index.scss new file mode 100644 index 0000000..ccf52c3 --- /dev/null +++ b/src/newsreader/scss/pages/integrations/index.scss @@ -0,0 +1,5 @@ +#integrations--page { + .section { + width: 70%; + } +} diff --git a/src/newsreader/scss/partials/_colors.scss b/src/newsreader/scss/partials/_colors.scss index b2f124d..87f6e49 100644 --- a/src/newsreader/scss/partials/_colors.scss +++ b/src/newsreader/scss/partials/_colors.scss @@ -12,6 +12,7 @@ $font-color: rgba(48, 51, 53, 1); $header-color: rgba(100, 101, 102, 1); $reddit-orange: rgba(255, 69, 0, 1); +$twitter-blue: rgba(29, 155, 240, 1); $transparant-red: transparentize($red, 0.8); $transparant-blue: transparentize($blue, 0.8); diff --git a/src/newsreader/templates/components/form/form.html b/src/newsreader/templates/components/form/form.html index e183c25..9f1ab47 100644 --- a/src/newsreader/templates/components/form/form.html +++ b/src/newsreader/templates/components/form/form.html @@ -4,7 +4,7 @@ {% csrf_token %} {% if title %} - {% include "components/form/title.html" with title=title only %} + {% include "components/header/header.html" with title=title only %} {% endif %} {% block intro %} diff --git a/src/newsreader/templates/components/form/title.html b/src/newsreader/templates/components/form/title.html deleted file mode 100644 index 8d6ded7..0000000 --- a/src/newsreader/templates/components/form/title.html +++ /dev/null @@ -1,3 +0,0 @@ -
-

{{ title }}

-
diff --git a/src/newsreader/templates/components/header/header.html b/src/newsreader/templates/components/header/header.html new file mode 100644 index 0000000..c21c233 --- /dev/null +++ b/src/newsreader/templates/components/header/header.html @@ -0,0 +1,3 @@ +
+

{{ title }}

+
From ead757589954cec2738ae0a66f4d8052270ee41d Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 14 Sep 2020 23:34:48 +0200 Subject: [PATCH 25/69] Update callback urls --- src/newsreader/conf/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/newsreader/conf/base.py b/src/newsreader/conf/base.py index 94ce2be..d41f352 100644 --- a/src/newsreader/conf/base.py +++ b/src/newsreader/conf/base.py @@ -201,12 +201,16 @@ VERSION = get_current_version() # Reddit integration REDDIT_CLIENT_ID = "CLIENT_ID" REDDIT_CLIENT_SECRET = "CLIENT_SECRET" -REDDIT_REDIRECT_URL = "http://127.0.0.1:8000/accounts/settings/reddit/callback/" +REDDIT_REDIRECT_URL = ( + "http://127.0.0.1:8000/accounts/settings/integrations/reddit/callback/" +) # Twitter integration TWITTER_CONSUMER_ID = "CONSUMER_ID" TWITTER_CONSUMER_SECRET = "CONSUMER_SECRET" -TWITTER_REDIRECT_URL = "http://127.0.0.1:8000/accounts/settings/twitter/callback/" +TWITTER_REDIRECT_URL = ( + "http://127.0.0.1:8000/accounts/settings/integrations/twitter/callback/" +) # Third party settings AXES_HANDLER = "axes.handlers.cache.AxesCacheHandler" From cfc48cea0543e5a71b5d9aec14532a00ab9a4944 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Tue, 15 Sep 2020 09:40:50 +0200 Subject: [PATCH 26/69] Rename integration buttons --- .../templates/accounts/views/integrations.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html index 3ef2134..e84d733 100644 --- a/src/newsreader/accounts/templates/accounts/views/integrations.html +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -10,15 +10,15 @@

Reddit

@@ -26,15 +26,15 @@

Twitter

From 0cf81e68d3ba30942cc42189e5cd6ad155d81d3e Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Wed, 16 Sep 2020 22:43:29 +0200 Subject: [PATCH 27/69] Move existing integration page tests --- .../accounts/tests/test_integrations.py | 124 +++++++++++++++++ .../accounts/tests/test_settings.py | 129 ------------------ 2 files changed, 124 insertions(+), 129 deletions(-) create mode 100644 src/newsreader/accounts/tests/test_integrations.py diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py new file mode 100644 index 0000000..c657585 --- /dev/null +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -0,0 +1,124 @@ +from unittest.mock import patch +from urllib.parse import urlencode +from uuid import uuid4 + +from django.core.cache import cache +from django.test import TestCase +from django.urls import reverse + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.news.collection.exceptions import StreamTooManyException + + +class RedditTemplateViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.base_url = reverse("accounts:reddit-template") + self.state = str(uuid4()) + + self.patch = patch("newsreader.news.collection.reddit.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + patch.stopall() + + def test_simple(self): + response = self.client.get(self.base_url) + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Return to integrations page") + + def test_successful_authorization(self): + self.mocked_post.return_value.json.return_value = { + "access_token": "1001010412", + "refresh_token": "134510143", + } + + cache.set(f"{self.user.email}-reddit-auth", self.state) + + params = {"state": self.state, "code": "Valid code"} + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.mocked_post.assert_called_once() + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Your reddit account was successfully linked.") + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, "1001010412") + self.assertEquals(self.user.reddit_refresh_token, "134510143") + + self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), None) + + def test_error(self): + params = {"error": "Denied authorization"} + + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Denied authorization") + + def test_invalid_state(self): + cache.set(f"{self.user.email}-reddit-auth", str(uuid4())) + + params = {"code": "Valid code", "state": "Invalid state"} + + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.assertEquals(response.status_code, 200) + self.assertContains( + response, "The saved state for Reddit authorization did not match" + ) + + def test_stream_error(self): + self.mocked_post.side_effect = StreamTooManyException + + cache.set(f"{self.user.email}-reddit-auth", self.state) + + params = {"state": self.state, "code": "Valid code"} + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.mocked_post.assert_called_once() + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Too many requests") + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, None) + self.assertEquals(self.user.reddit_refresh_token, None) + + self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) + + def test_unexpected_json(self): + self.mocked_post.return_value.json.return_value = {"message": "Happy eastern"} + + cache.set(f"{self.user.email}-reddit-auth", self.state) + + params = {"state": self.state, "code": "Valid code"} + url = f"{self.base_url}?{urlencode(params)}" + + response = self.client.get(url) + + self.mocked_post.assert_called_once() + + self.assertEquals(response.status_code, 200) + self.assertContains(response, "Access and refresh token not found in response") + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, None) + self.assertEquals(self.user.reddit_refresh_token, None) + + self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) diff --git a/src/newsreader/accounts/tests/test_settings.py b/src/newsreader/accounts/tests/test_settings.py index d093ea4..57d9486 100644 --- a/src/newsreader/accounts/tests/test_settings.py +++ b/src/newsreader/accounts/tests/test_settings.py @@ -1,14 +1,9 @@ -from unittest.mock import patch -from urllib.parse import urlencode -from uuid import uuid4 -from django.core.cache import cache from django.test import TestCase from django.urls import reverse from newsreader.accounts.models import User from newsreader.accounts.tests.factories import UserFactory -from newsreader.news.collection.exceptions import StreamTooManyException class SettingsViewTestCase(TestCase): @@ -22,7 +17,6 @@ class SettingsViewTestCase(TestCase): response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertContains(response, "Authorize Reddit account") def test_user_credential_change(self): response = self.client.post( @@ -36,126 +30,3 @@ class SettingsViewTestCase(TestCase): self.assertEquals(user.first_name, "First name") self.assertEquals(user.last_name, "Last name") - - def test_linked_reddit_account(self): - self.user.reddit_refresh_token = "test" - self.user.save() - - response = self.client.get(self.url) - - self.assertEquals(response.status_code, 200) - self.assertNotContains(response, "Authorize Reddit account") - - -class RedditTemplateViewTestCase(TestCase): - def setUp(self): - self.user = UserFactory(email="test@test.nl", password="test") - self.client.force_login(self.user) - - self.base_url = reverse("accounts:reddit-template") - self.state = str(uuid4()) - - self.patch = patch("newsreader.news.collection.reddit.post") - self.mocked_post = self.patch.start() - - def tearDown(self): - patch.stopall() - - def test_simple(self): - response = self.client.get(self.base_url) - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Return to settings page") - - def test_successful_authorization(self): - self.mocked_post.return_value.json.return_value = { - "access_token": "1001010412", - "refresh_token": "134510143", - } - - cache.set(f"{self.user.email}-reddit-auth", self.state) - - params = {"state": self.state, "code": "Valid code"} - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.mocked_post.assert_called_once() - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Your reddit account was successfully linked.") - - self.user.refresh_from_db() - - self.assertEquals(self.user.reddit_access_token, "1001010412") - self.assertEquals(self.user.reddit_refresh_token, "134510143") - - self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), None) - - def test_error(self): - params = {"error": "Denied authorization"} - - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Denied authorization") - - def test_invalid_state(self): - cache.set(f"{self.user.email}-reddit-auth", str(uuid4())) - - params = {"code": "Valid code", "state": "Invalid state"} - - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.assertEquals(response.status_code, 200) - self.assertContains( - response, "The saved state for Reddit authorization did not match" - ) - - def test_stream_error(self): - self.mocked_post.side_effect = StreamTooManyException - - cache.set(f"{self.user.email}-reddit-auth", self.state) - - params = {"state": self.state, "code": "Valid code"} - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.mocked_post.assert_called_once() - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Too many requests") - - self.user.refresh_from_db() - - self.assertEquals(self.user.reddit_access_token, None) - self.assertEquals(self.user.reddit_refresh_token, None) - - self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) - - def test_unexpected_json(self): - self.mocked_post.return_value.json.return_value = {"message": "Happy eastern"} - - cache.set(f"{self.user.email}-reddit-auth", self.state) - - params = {"state": self.state, "code": "Valid code"} - url = f"{self.base_url}?{urlencode(params)}" - - response = self.client.get(url) - - self.mocked_post.assert_called_once() - - self.assertEquals(response.status_code, 200) - self.assertContains(response, "Access and refresh token not found in response") - - self.user.refresh_from_db() - - self.assertEquals(self.user.reddit_access_token, None) - self.assertEquals(self.user.reddit_refresh_token, None) - - self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) From cdc7ca90dabf48520e7680e12f23db8cc3336894 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Wed, 16 Sep 2020 22:54:10 +0200 Subject: [PATCH 28/69] Update reddit template --- .../accounts/templates/accounts/views/reddit.html | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/newsreader/accounts/templates/accounts/views/reddit.html b/src/newsreader/accounts/templates/accounts/views/reddit.html index b393bbe..76c6115 100644 --- a/src/newsreader/accounts/templates/accounts/views/reddit.html +++ b/src/newsreader/accounts/templates/accounts/views/reddit.html @@ -1,17 +1,20 @@ {% extends "base.html" %} +{% load i18n %} {% block content %} -
+
{% if error %} -

Reddit authorization failed

+

{% trans "Reddit authorization failed" %}

{{ error }}

{% elif access_token and refresh_token %} -

Reddit account is linked

-

Your reddit account was successfully linked.

+

{% trans "Reddit account is linked" %}

+

{% trans "Your reddit account was successfully linked." %}

{% endif %} -

Return to settings page

+

+ {% trans "Return to integrations page" %} +

{% endblock %} From 655e641213d40567b5c4d8f495f1a5828d1524bc Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Thu, 17 Sep 2020 21:37:02 +0200 Subject: [PATCH 29/69] Add revoking reddit account access --- .../accounts/views/integrations.html | 51 +++++++++++++------ .../templates/accounts/views/reddit.html | 2 +- src/newsreader/accounts/urls.py | 6 +++ src/newsreader/accounts/views/__init__.py | 1 + src/newsreader/accounts/views/integrations.py | 45 +++++++++++++++- src/newsreader/news/collection/reddit.py | 15 ++++++ 6 files changed, 102 insertions(+), 18 deletions(-) diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html index e84d733..81a1fa1 100644 --- a/src/newsreader/accounts/templates/accounts/views/integrations.html +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -9,33 +9,52 @@

Reddit

- + {% if reddit_authorization_url %} + + {% trans "Authorize account" %} + + {% else %} + + {% endif %} - + {% if reddit_refresh_url %} + + {% trans "Refresh token" %} + + {% else %} + + {% endif %} - + {% if reddit_revoke_url %} + + {% trans "Deauthorize account" %} + + {% else %} + + {% endif %}
+

Twitter

- - + -
diff --git a/src/newsreader/accounts/templates/accounts/views/reddit.html b/src/newsreader/accounts/templates/accounts/views/reddit.html index 76c6115..5d4f539 100644 --- a/src/newsreader/accounts/templates/accounts/views/reddit.html +++ b/src/newsreader/accounts/templates/accounts/views/reddit.html @@ -13,7 +13,7 @@ {% endif %}

- {% trans "Return to integrations page" %} + {% trans "Return to integrations page" %}

diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index c250a0f..68f27a6 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -13,6 +13,7 @@ from newsreader.accounts.views import ( PasswordResetConfirmView, PasswordResetDoneView, PasswordResetView, + RedditRevokeRedirectView, RedditTemplateView, RedditTokenRedirectView, RegistrationClosedView, @@ -78,6 +79,11 @@ urlpatterns = [ login_required(RedditTokenRedirectView.as_view()), name="reddit-refresh", ), + path( + "settings/integrations/reddit/revoke/", + login_required(RedditRevokeRedirectView.as_view()), + name="reddit-revoke", + ), path( "settings/integrations", login_required(IntegrationsView.as_view()), diff --git a/src/newsreader/accounts/views/__init__.py b/src/newsreader/accounts/views/__init__.py index 296cb28..9325728 100644 --- a/src/newsreader/accounts/views/__init__.py +++ b/src/newsreader/accounts/views/__init__.py @@ -1,6 +1,7 @@ from newsreader.accounts.views.auth import LoginView, LogoutView from newsreader.accounts.views.integrations import ( IntegrationsView, + RedditRevokeRedirectView, RedditTemplateView, RedditTokenRedirectView, ) diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index a32c747..fa343a2 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -1,3 +1,5 @@ +import logging + from django.contrib import messages from django.core.cache import cache from django.urls import reverse_lazy @@ -8,10 +10,14 @@ from newsreader.news.collection.exceptions import StreamException from newsreader.news.collection.reddit import ( get_reddit_access_token, get_reddit_authorization_url, + revoke_reddit_token, ) from newsreader.news.collection.tasks import RedditTokenTask +logger = logging.getLogger(__name__) + + class IntegrationsView(TemplateView): template_name = "accounts/views/integrations.html" @@ -41,6 +47,11 @@ class IntegrationsView(TemplateView): return { "reddit_authorization_url": reddit_authorization_url, "reddit_refresh_url": reddit_refresh_url, + "reddit_revoke_url": ( + reverse_lazy("accounts:reddit-revoke") + if not reddit_authorization_url + else None + ), } @@ -89,7 +100,7 @@ class RedditTemplateView(TemplateView): class RedditTokenRedirectView(RedirectView): - url = reverse_lazy("accounts:settings") + url = reverse_lazy("accounts:integrations") def get(self, request, *args, **kwargs): response = super().get(request, *args, **kwargs) @@ -105,3 +116,35 @@ class RedditTokenRedirectView(RedirectView): messages.error(request, _("Unable to retrieve token")) return response + + +class RedditRevokeRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + response = super().get(request, *args, **kwargs) + + user = request.user + + if not user.reddit_refresh_token: + messages.error(request, _("No reddit account is linked to this account")) + return response + + try: + is_revoked = revoke_reddit_token(user) + except StreamException: + logger.exception(f"Unable to revoke reddit token for {user.pk}") + + messages.error(request, _("Unable to revoke reddit token")) + return response + + if not is_revoked: + messages.error(request, _("Unable to revoke reddit token")) + return response + + user.reddit_access_token = None + user.reddit_refresh_token = None + user.save() + + messages.success(request, _("Reddit account deathorized")) + return response diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 0122bb1..1cb9e29 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -98,6 +98,21 @@ def get_reddit_access_token(code, user): return response_data["access_token"], response_data["refresh_token"] +# Note that the API always returns 204's with correct basic auth headers +def revoke_reddit_token(user): + client_auth = requests.auth.HTTPBasicAuth( + settings.REDDIT_CLIENT_ID, settings.REDDIT_CLIENT_SECRET + ) + + response = post( + f"{REDDIT_URL}/api/v1/revoke_token", + data={"token": user.reddit_refresh_token, "token_type_hint": "refresh_token"}, + auth=client_auth, + ) + + return response.status_code == 204 + + class RedditBuilder(PostBuilder): rule_type = RuleTypeChoices.subreddit From 1fd4b60710e0fafa328e7821aeb02cf8172d5961 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Thu, 17 Sep 2020 22:53:07 +0200 Subject: [PATCH 30/69] Add TODO's --- src/newsreader/accounts/views/integrations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index fa343a2..94abe7a 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -18,6 +18,7 @@ from newsreader.news.collection.tasks import RedditTokenTask logger = logging.getLogger(__name__) +# TODO add tests class IntegrationsView(TemplateView): template_name = "accounts/views/integrations.html" @@ -99,6 +100,7 @@ class RedditTemplateView(TemplateView): ) +# TODO add tests class RedditTokenRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") @@ -118,6 +120,7 @@ class RedditTokenRedirectView(RedirectView): return response +# TODO add tests class RedditRevokeRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") From eb661b8abce1f6198dcd60bf6f70a0044af08cd4 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 18 Sep 2020 21:25:15 +0200 Subject: [PATCH 31/69] Add reddit refresh token view tests --- .../accounts/tests/test_integrations.py | 49 +++++++++++++++++++ src/newsreader/accounts/views/integrations.py | 2 - 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py index c657585..710161c 100644 --- a/src/newsreader/accounts/tests/test_integrations.py +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -1,3 +1,4 @@ +from unittest import skip from unittest.mock import patch from urllib.parse import urlencode from uuid import uuid4 @@ -122,3 +123,51 @@ class RedditTemplateViewTestCase(TestCase): self.assertEquals(self.user.reddit_refresh_token, None) self.assertEquals(cache.get(f"{self.user.email}-reddit-auth"), self.state) + + +class RedditTokenRedirectViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.RedditTokenTask") + self.mocked_task = self.patch.start() + + def tearDown(self): + cache.clear() + + def test_simple(self): + response = self.client.get(reverse("accounts:reddit-refresh")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_task.delay.assert_called_once_with(self.user.pk) + + self.assertEquals(1, cache.get(f"{self.user.email}-reddit-refresh")) + + def test_not_active(self): + cache.set(f"{self.user.email}-reddit-refresh", 1) + + response = self.client.get(reverse("accounts:reddit-refresh")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_task.delay.assert_not_called() + + +class RedditRevokeRedirectViewTestCase(TestCase): + @skip("Not implemented") + def test_simple(self): + pass + + @skip("Not implemented") + def test_no_refresh_token(self): + pass + + @skip("Not implemented") + def test_unsuccessful_response(self): + pass + + @skip("Not implemented") + def test_stream_exception(self): + pass diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 94abe7a..3fecf32 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -100,7 +100,6 @@ class RedditTemplateView(TemplateView): ) -# TODO add tests class RedditTokenRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") @@ -120,7 +119,6 @@ class RedditTokenRedirectView(RedirectView): return response -# TODO add tests class RedditRevokeRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") From 5743dd7096cd2ab2dd8f293cb0c46d69e445f9b2 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 18 Sep 2020 21:39:59 +0200 Subject: [PATCH 32/69] Add reddit revoke view tests --- .../accounts/tests/test_integrations.py | 73 ++++++++++++++++--- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py index 710161c..ae1eef0 100644 --- a/src/newsreader/accounts/tests/test_integrations.py +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -1,4 +1,3 @@ -from unittest import skip from unittest.mock import patch from urllib.parse import urlencode from uuid import uuid4 @@ -8,7 +7,10 @@ from django.test import TestCase from django.urls import reverse from newsreader.accounts.tests.factories import UserFactory -from newsreader.news.collection.exceptions import StreamTooManyException +from newsreader.news.collection.exceptions import ( + StreamException, + StreamTooManyException, +) class RedditTemplateViewTestCase(TestCase): @@ -156,18 +158,69 @@ class RedditTokenRedirectViewTestCase(TestCase): class RedditRevokeRedirectViewTestCase(TestCase): - @skip("Not implemented") + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.revoke_reddit_token") + self.mocked_revoke = self.patch.start() + def test_simple(self): - pass + self.user.reddit_access_token = "jadajadajada" + self.user.reddit_refresh_token = "jadajadajada" + self.user.save() + + self.mocked_revoke.return_value = True + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_revoke.assert_called_once_with(self.user) + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, None) + self.assertEquals(self.user.reddit_refresh_token, None) - @skip("Not implemented") def test_no_refresh_token(self): - pass + self.user.reddit_refresh_token = None + self.user.save() + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_revoke.assert_not_called() - @skip("Not implemented") def test_unsuccessful_response(self): - pass + self.user.reddit_access_token = "jadajadajada" + self.user.reddit_refresh_token = "jadajadajada" + self.user.save() + + self.mocked_revoke.return_value = False + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, "jadajadajada") + self.assertEquals(self.user.reddit_refresh_token, "jadajadajada") - @skip("Not implemented") def test_stream_exception(self): - pass + self.user.reddit_access_token = "jadajadajada" + self.user.reddit_refresh_token = "jadajadajada" + self.user.save() + + self.mocked_revoke.side_effect = StreamException + + response = self.client.get(reverse("accounts:reddit-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.reddit_access_token, "jadajadajada") + self.assertEquals(self.user.reddit_refresh_token, "jadajadajada") From 72f78f89aea318b843123af4c6e3c0b5d493b643 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 18 Sep 2020 23:22:25 +0200 Subject: [PATCH 33/69] Add simple integration tests --- .../accounts/tests/test_integrations.py | 49 +++++++++++++++++++ src/newsreader/accounts/views/integrations.py | 1 - 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py index ae1eef0..098a126 100644 --- a/src/newsreader/accounts/tests/test_integrations.py +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -6,6 +6,8 @@ from django.core.cache import cache from django.test import TestCase from django.urls import reverse +from bs4 import BeautifulSoup + from newsreader.accounts.tests.factories import UserFactory from newsreader.news.collection.exceptions import ( StreamException, @@ -13,6 +15,53 @@ from newsreader.news.collection.exceptions import ( ) +class IntegrationsViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.url = reverse("accounts:integrations") + + +class RedditIntegrationsTestCase(IntegrationsViewTestCase): + def test_reddit_authorization(self): + self.user.reddit_refresh_token = None + self.user.save() + + response = self.client.get(self.url) + + soup = BeautifulSoup(response.content, features="lxml") + button = soup.find("a", class_="link button button--reddit") + + self.assertEquals(button.text.strip(), "Authorize account") + + def test_reddit_refresh_token(self): + self.user.reddit_refresh_token = "jadajadajada" + self.user.reddit_access_token = None + self.user.save() + + response = self.client.get(self.url) + + soup = BeautifulSoup(response.content, features="lxml") + button = soup.find("a", class_="link button button--reddit") + + self.assertEquals(button.text.strip(), "Refresh token") + + def test_reddit_revoke(self): + self.user.reddit_refresh_token = "jadajadajada" + self.user.reddit_access_token = None + self.user.save() + + response = self.client.get(self.url) + + soup = BeautifulSoup(response.content, features="lxml") + buttons = soup.find_all("a", class_="link button button--reddit") + + self.assertIn( + "Deauthorize account", [button.text.strip() for button in buttons] + ) + + class RedditTemplateViewTestCase(TestCase): def setUp(self): self.user = UserFactory(email="test@test.nl", password="test") diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 3fecf32..fa343a2 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -18,7 +18,6 @@ from newsreader.news.collection.tasks import RedditTokenTask logger = logging.getLogger(__name__) -# TODO add tests class IntegrationsView(TemplateView): template_name = "accounts/views/integrations.html" From 355f16b387cd2256722490f2d9800afb3a180915 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 18 Sep 2020 23:24:20 +0200 Subject: [PATCH 34/69] Remove usesless meta classes --- src/newsreader/news/collection/base.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index 1377078..eb11619 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -24,9 +24,6 @@ class Stream: def parse(self, response): raise NotImplementedError - class Meta: - abstract = True - class Client: """ @@ -47,9 +44,6 @@ class Client: def __exit__(self, *args, **kwargs): pass - class Meta: - abstract = True - class Builder: """ @@ -85,9 +79,6 @@ class Builder: strip_comments=True, ) - class Meta: - abstract = True - class Collector: """ @@ -104,9 +95,6 @@ class Collector: def collect(self, rules=None): raise NotImplementedError - class Meta: - abstract = True - class Scheduler: """ @@ -134,9 +122,6 @@ class PostBuilder(Builder): for post in self.instances: post.save() - class Meta: - abstract = True - class PostStream(Stream): rule_type = None From abb5328feba0a0f9dc42b2b88b2c7535e12de773 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 19 Sep 2020 17:00:41 +0200 Subject: [PATCH 35/69] Form / view file refactor --- .../js/pages/homepage/components/PostModal.js | 2 + src/newsreader/js/pages/homepage/constants.js | 1 + .../news/collection/forms/__init__.py | 4 ++ src/newsreader/news/collection/forms/base.py | 30 +++++++++ src/newsreader/news/collection/forms/feed.py | 28 +++++++++ .../collection/{forms.py => forms/reddit.py} | 54 +--------------- src/newsreader/news/collection/forms/rules.py | 15 +++++ .../news/collection/forms/twitter.py | 32 ++++++++++ src/newsreader/news/collection/models.py | 2 +- .../{rule-create.html => feed-create.html} | 2 +- .../{rule-update.html => feed-update.html} | 6 +- .../news/collection/views/import.html | 2 +- .../news/collection/views/rules.html | 5 +- .../views/twitter/timeline-create.html | 9 +++ .../views/twitter/timeline-update.html | 14 +++++ .../news/collection/tests/views/base.py | 2 +- .../news/collection/tests/views/test_crud.py | 10 +-- .../tests/views/test_import_view.py | 4 +- src/newsreader/news/collection/urls.py | 40 +++++++----- .../news/collection/views/__init__.py | 12 +++- src/newsreader/news/collection/views/base.py | 2 - src/newsreader/news/collection/views/feed.py | 62 +++++++++++++++++++ .../news/collection/views/reddit.py | 6 +- src/newsreader/news/collection/views/rules.py | 54 +--------------- .../news/collection/views/twitter.py | 29 +++++++++ src/newsreader/utils/opml.py | 1 + 26 files changed, 285 insertions(+), 143 deletions(-) create mode 100644 src/newsreader/news/collection/forms/__init__.py create mode 100644 src/newsreader/news/collection/forms/base.py create mode 100644 src/newsreader/news/collection/forms/feed.py rename src/newsreader/news/collection/{forms.py => forms/reddit.py} (51%) create mode 100644 src/newsreader/news/collection/forms/rules.py create mode 100644 src/newsreader/news/collection/forms/twitter.py rename src/newsreader/news/collection/templates/news/collection/views/{rule-create.html => feed-create.html} (78%) rename src/newsreader/news/collection/templates/news/collection/views/{rule-update.html => feed-update.html} (72%) create mode 100644 src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html create mode 100644 src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html create mode 100644 src/newsreader/news/collection/views/feed.py create mode 100644 src/newsreader/news/collection/views/twitter.py diff --git a/src/newsreader/js/pages/homepage/components/PostModal.js b/src/newsreader/js/pages/homepage/components/PostModal.js index 08033bc..3d7a888 100644 --- a/src/newsreader/js/pages/homepage/components/PostModal.js +++ b/src/newsreader/js/pages/homepage/components/PostModal.js @@ -44,6 +44,8 @@ class PostModal extends React.Component { const post = this.props.post; const publicationDate = formatDatetime(post.publicationDate); const titleClassName = post.read ? 'post__title post__title--read' : 'post__title'; + + // TODO add mapping & get urls from backend const ruleUrl = this.props.rule.type === FEED ? `/collection/rules/${this.props.rule.id}/` diff --git a/src/newsreader/js/pages/homepage/constants.js b/src/newsreader/js/pages/homepage/constants.js index 66b6365..22184b9 100644 --- a/src/newsreader/js/pages/homepage/constants.js +++ b/src/newsreader/js/pages/homepage/constants.js @@ -3,3 +3,4 @@ export const CATEGORY_TYPE = 'CATEGORY'; export const SUBREDDIT = 'subreddit'; export const FEED = 'feed'; +export const TWITTER_TIMELINE = 'twitter_timeline'; diff --git a/src/newsreader/news/collection/forms/__init__.py b/src/newsreader/news/collection/forms/__init__.py new file mode 100644 index 0000000..88a51c7 --- /dev/null +++ b/src/newsreader/news/collection/forms/__init__.py @@ -0,0 +1,4 @@ +from newsreader.news.collection.forms.feed import FeedForm, OPMLImportForm +from newsreader.news.collection.forms.reddit import SubRedditForm +from newsreader.news.collection.forms.rules import CollectionRuleBulkForm +from newsreader.news.collection.forms.twitter import TwitterTimelineForm diff --git a/src/newsreader/news/collection/forms/base.py b/src/newsreader/news/collection/forms/base.py new file mode 100644 index 0000000..0826919 --- /dev/null +++ b/src/newsreader/news/collection/forms/base.py @@ -0,0 +1,30 @@ +from django import forms + + +from newsreader.news.collection.models import CollectionRule +from newsreader.news.core.models import Category + + +class CollectionRuleForm(forms.ModelForm): + category = forms.ModelChoiceField(required=False, queryset=Category.objects.all()) + + def __init__(self, *args, **kwargs): + self.user = kwargs.pop("user") + + super().__init__(*args, **kwargs) + + self.fields["category"].queryset = Category.objects.filter(user=self.user) + + def save(self, commit=True): + instance = super().save(commit=False) + instance.user = self.user + + if commit: + instance.save() + self.save_m2m() + + return instance + + class Meta: + model = CollectionRule + fields = "__all__" diff --git a/src/newsreader/news/collection/forms/feed.py b/src/newsreader/news/collection/forms/feed.py new file mode 100644 index 0000000..4a22a2e --- /dev/null +++ b/src/newsreader/news/collection/forms/feed.py @@ -0,0 +1,28 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +import pytz + +from newsreader.core.forms import CheckboxInput +from newsreader.news.collection.forms.base import CollectionRuleForm +from newsreader.news.collection.models import CollectionRule + + +class FeedForm(CollectionRuleForm): + timezone = forms.ChoiceField( + widget=forms.Select(attrs={"size": len(pytz.all_timezones)}), + choices=((timezone, timezone) for timezone in pytz.all_timezones), + help_text=_("The timezone which the feed uses"), + initial=pytz.utc, + ) + + class Meta: + model = CollectionRule + fields = ("name", "url", "timezone", "favicon", "category") + + +class OPMLImportForm(forms.Form): + file = forms.FileField(allow_empty_file=False) + skip_existing = forms.BooleanField( + initial=False, required=False, widget=CheckboxInput + ) diff --git a/src/newsreader/news/collection/forms.py b/src/newsreader/news/collection/forms/reddit.py similarity index 51% rename from src/newsreader/news/collection/forms.py rename to src/newsreader/news/collection/forms/reddit.py index c79a867..0bcde9f 100644 --- a/src/newsreader/news/collection/forms.py +++ b/src/newsreader/news/collection/forms/reddit.py @@ -9,6 +9,7 @@ from newsreader.core.forms import CheckboxInput from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.reddit import REDDIT_API_URL +from newsreader.news.collection.forms.base import CollectionRuleForm from newsreader.news.core.models import Category @@ -22,53 +23,9 @@ def get_reddit_help_text(): ) -class CollectionRuleForm(forms.ModelForm): - category = forms.ModelChoiceField(required=False, queryset=Category.objects.all()) - timezone = forms.ChoiceField( - widget=forms.Select(attrs={"size": len(pytz.all_timezones)}), - choices=((timezone, timezone) for timezone in pytz.all_timezones), - help_text=_("The timezone which the feed uses"), - initial=pytz.utc, - ) - - def __init__(self, *args, **kwargs): - self.user = kwargs.pop("user") - - super().__init__(*args, **kwargs) - - self.fields["category"].queryset = Category.objects.filter(user=self.user) - - def save(self, commit=True): - instance = super().save(commit=False) - instance.user = self.user - - if commit: - instance.save() - self.save_m2m() - - return instance - - class Meta: - model = CollectionRule - fields = ("name", "url", "timezone", "favicon", "category") - - -class CollectionRuleBulkForm(forms.Form): - rules = forms.ModelMultipleChoiceField(queryset=CollectionRule.objects.none()) - - def __init__(self, user, *args, **kwargs): - self.user = user - - super().__init__(*args, **kwargs) - - self.fields["rules"].queryset = CollectionRule.objects.filter(user=user) - - -class SubRedditRuleForm(CollectionRuleForm): +class SubRedditForm(CollectionRuleForm): url = forms.URLField(max_length=1024, help_text=get_reddit_help_text) - timezone = None - def clean_url(self): url = self.cleaned_data["url"] @@ -92,10 +49,3 @@ class SubRedditRuleForm(CollectionRuleForm): class Meta: model = CollectionRule fields = ("name", "url", "favicon", "category") - - -class OPMLImportForm(forms.Form): - file = forms.FileField(allow_empty_file=False) - skip_existing = forms.BooleanField( - initial=False, required=False, widget=CheckboxInput - ) diff --git a/src/newsreader/news/collection/forms/rules.py b/src/newsreader/news/collection/forms/rules.py new file mode 100644 index 0000000..e5dccf9 --- /dev/null +++ b/src/newsreader/news/collection/forms/rules.py @@ -0,0 +1,15 @@ +from django import forms + + +from newsreader.news.collection.models import CollectionRule + + +class CollectionRuleBulkForm(forms.Form): + rules = forms.ModelMultipleChoiceField(queryset=CollectionRule.objects.none()) + + def __init__(self, user, *args, **kwargs): + self.user = user + + super().__init__(*args, **kwargs) + + self.fields["rules"].queryset = CollectionRule.objects.filter(user=user) diff --git a/src/newsreader/news/collection/forms/twitter.py b/src/newsreader/news/collection/forms/twitter.py new file mode 100644 index 0000000..441b243 --- /dev/null +++ b/src/newsreader/news/collection/forms/twitter.py @@ -0,0 +1,32 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +import pytz + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms.base import CollectionRuleForm +from newsreader.news.collection.models import CollectionRule + + +class TwitterTimelineForm(CollectionRuleForm): + screen_name = forms.CharField( + max_length=255, + label=_("Twitter profile name"), + help_text=_("Profile name without hashtags"), + ) + + def save(self, commit=True): + instance = super().save(commit=False) + + instance.type = RuleTypeChoices.twitter_timeline + instance.timezone = str(pytz.utc) + + if commit: + instance.save() + self.save_m2m() + + return instance + + class Meta: + model = CollectionRule + fields = ("name", "screen_name", "favicon", "category") diff --git a/src/newsreader/news/collection/models.py b/src/newsreader/news/collection/models.py index 0cbfa22..d1d8024 100644 --- a/src/newsreader/news/collection/models.py +++ b/src/newsreader/news/collection/models.py @@ -70,4 +70,4 @@ class CollectionRule(TimeStampedModel): if self.type == RuleTypeChoices.subreddit: return reverse("news:collection:subreddit-update", kwargs={"pk": self.pk}) - return reverse("news:collection:rule-update", kwargs={"pk": self.pk}) + return reverse("news:collection:feed-update", kwargs={"pk": self.pk}) diff --git a/src/newsreader/news/collection/templates/news/collection/views/rule-create.html b/src/newsreader/news/collection/templates/news/collection/views/feed-create.html similarity index 78% rename from src/newsreader/news/collection/templates/news/collection/views/rule-create.html rename to src/newsreader/news/collection/templates/news/collection/views/feed-create.html index 82ed6c5..c24791a 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rule-create.html +++ b/src/newsreader/news/collection/templates/news/collection/views/feed-create.html @@ -4,6 +4,6 @@ {% block content %}
{% url "news:collection:rules" as cancel_url %} - {% include "components/form/form.html" with form=form title="Create rule" cancel_url=cancel_url confirm_text="Create rule" %} + {% include "components/form/form.html" with form=form title="Add a feed" cancel_url=cancel_url confirm_text="Add feed" %}
{% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/rule-update.html b/src/newsreader/news/collection/templates/news/collection/views/feed-update.html similarity index 72% rename from src/newsreader/news/collection/templates/news/collection/views/rule-update.html rename to src/newsreader/news/collection/templates/news/collection/views/feed-update.html index 0a705b8..33b1faf 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rule-update.html +++ b/src/newsreader/news/collection/templates/news/collection/views/feed-update.html @@ -3,12 +3,12 @@ {% block content %}
- {% if rule.error %} + {% if feed.error %} {% trans "Failed to retrieve posts" as title %} - {% include "components/textbox/textbox.html" with title=title body=rule.error class="text-section--error" only %} + {% include "components/textbox/textbox.html" with title=title body=feed.error class="text-section--error" only %} {% endif %} {% url "news:collection:rules" as cancel_url %} - {% include "components/form/form.html" with form=form title="Update rule" cancel_url=cancel_url confirm_text="Save rule" only %} + {% include "components/form/form.html" with form=form title="Update feed" cancel_url=cancel_url confirm_text="Save feed" only %}
{% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/import.html b/src/newsreader/news/collection/templates/news/collection/views/import.html index df19887..9719847 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/import.html +++ b/src/newsreader/news/collection/templates/news/collection/views/import.html @@ -4,6 +4,6 @@ {% block content %}
{% url "news:collection:rules" as cancel_url %} - {% include "components/form/form.html" with form=form title="Import an OPML file" cancel_url=cancel_url confirm_text="Import rules" %} + {% include "components/form/form.html" with form=form title="Import an OPML file" cancel_url=cancel_url confirm_text="Import feeds" %}
{% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/rules.html b/src/newsreader/news/collection/templates/news/collection/views/rules.html index 0cd1870..cd98b29 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rules.html +++ b/src/newsreader/news/collection/templates/news/collection/views/rules.html @@ -14,8 +14,9 @@
- {% trans "Add a rule" %} - {% trans "Add a subreddit" %} + {% trans "Add a feed" %} + {% trans "Add a subreddit" %} + {% trans "Import rules" %}
diff --git a/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html new file mode 100644 index 0000000..7c8eb13 --- /dev/null +++ b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-create.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% load static %} + +{% block content %} +
+ {% url "news:collection:rules" as cancel_url %} + {% include "components/form/form.html" with form=form title="Add a Twitter profile" cancel_url=cancel_url confirm_text="Add profile" %} +
+{% endblock %} diff --git a/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html new file mode 100644 index 0000000..51de47a --- /dev/null +++ b/src/newsreader/news/collection/templates/news/collection/views/twitter/timeline-update.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{% load static i18n %} + +{% block content %} +
+ {% if timeline.error %} + {% trans "Failed to retrieve posts" as title %} + {% include "components/textbox/textbox.html" with title=title body=timeline.error class="text-section--error" only %} + {% endif %} + + {% url "news:collection:rules" as cancel_url %} + {% include "components/form/form.html" with form=form title="Update profile" cancel_url=cancel_url confirm_text="Save profile" %} +
+{% endblock %} diff --git a/src/newsreader/news/collection/tests/views/base.py b/src/newsreader/news/collection/tests/views/base.py index d7de171..17f232c 100644 --- a/src/newsreader/news/collection/tests/views/base.py +++ b/src/newsreader/news/collection/tests/views/base.py @@ -49,7 +49,7 @@ class CollectionRuleViewTestCase: timezone=other_rule.timezone, ) - other_url = reverse("news:collection:rule-update", args=[other_rule.pk]) + other_url = reverse("news:collection:feed-update", args=[other_rule.pk]) response = self.client.post(other_url, self.form_data) self.assertEquals(response.status_code, 404) diff --git a/src/newsreader/news/collection/tests/views/test_crud.py b/src/newsreader/news/collection/tests/views/test_crud.py index 61f6835..e10f997 100644 --- a/src/newsreader/news/collection/tests/views/test_crud.py +++ b/src/newsreader/news/collection/tests/views/test_crud.py @@ -10,11 +10,11 @@ from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCa from newsreader.news.core.tests.factories import CategoryFactory -class CollectionRuleCreateViewTestCase(CollectionRuleViewTestCase, TestCase): +class FeedCreateViewTestCase(CollectionRuleViewTestCase, TestCase): def setUp(self): super().setUp() - self.url = reverse("news:collection:rule-create") + self.url = reverse("news:collection:feed-create") self.form_data.update( name="new rule", @@ -38,14 +38,14 @@ class CollectionRuleCreateViewTestCase(CollectionRuleViewTestCase, TestCase): self.assertEquals(rule.user.pk, self.user.pk) -class CollectionRuleUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): +class FeedUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): def setUp(self): super().setUp() self.rule = FeedFactory( name="collection rule", user=self.user, category=self.category ) - self.url = reverse("news:collection:rule-update", kwargs={"pk": self.rule.pk}) + self.url = reverse("news:collection:feed-update", kwargs={"pk": self.rule.pk}) self.form_data.update( name=self.rule.name, @@ -94,7 +94,7 @@ class CollectionRuleUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): category=self.category, type=RuleTypeChoices.subreddit, ) - url = reverse("news:collection:rule-update", kwargs={"pk": rule.pk}) + url = reverse("news:collection:feed-update", kwargs={"pk": rule.pk}) response = self.client.get(url) diff --git a/src/newsreader/news/collection/tests/views/test_import_view.py b/src/newsreader/news/collection/tests/views/test_import_view.py index f4188e7..a1f0017 100644 --- a/src/newsreader/news/collection/tests/views/test_import_view.py +++ b/src/newsreader/news/collection/tests/views/test_import_view.py @@ -84,7 +84,7 @@ class OPMLImportTestCase(TestCase): rules = CollectionRule.objects.all() self.assertEquals(len(rules), 0) - self.assertFormError(response, "form", "file", _("No (new) rules found")) + self.assertFormError(response, "form", "file", _("No (new) feeds found")) def test_invalid_feeds(self): file_path = self._get_file_path("invalid-url-feeds.opml") @@ -99,7 +99,7 @@ class OPMLImportTestCase(TestCase): rules = CollectionRule.objects.all() self.assertEquals(len(rules), 0) - self.assertFormError(response, "form", "file", _("No (new) rules found")) + self.assertFormError(response, "form", "file", _("No (new) feeds found")) def test_invalid_file(self): file_path = self._get_file_path("test.png") diff --git a/src/newsreader/news/collection/urls.py b/src/newsreader/news/collection/urls.py index 5253210..7d883f2 100644 --- a/src/newsreader/news/collection/urls.py +++ b/src/newsreader/news/collection/urls.py @@ -11,12 +11,14 @@ from newsreader.news.collection.views import ( CollectionRuleBulkDeleteView, CollectionRuleBulkDisableView, CollectionRuleBulkEnableView, - CollectionRuleCreateView, CollectionRuleListView, - CollectionRuleUpdateView, + FeedCreateView, + FeedUpdateView, OPMLImportView, SubRedditCreateView, SubRedditUpdateView, + TwitterTimelineCreateView, + TwitterTimelineUpdateView, ) @@ -28,17 +30,13 @@ endpoints = [ ] urlpatterns = [ + # Feeds + path( + "feeds//", login_required(FeedUpdateView.as_view()), name="feed-update" + ), + path("feeds/create/", login_required(FeedCreateView.as_view()), name="feed-create"), + # Generic rules path("rules/", login_required(CollectionRuleListView.as_view()), name="rules"), - path( - "rules//", - login_required(CollectionRuleUpdateView.as_view()), - name="rule-update", - ), - path( - "rules/create/", - login_required(CollectionRuleCreateView.as_view()), - name="rule-create", - ), path( "rules/delete/", login_required(CollectionRuleBulkDeleteView.as_view()), @@ -54,15 +52,27 @@ urlpatterns = [ login_required(CollectionRuleBulkDisableView.as_view()), name="rules-disable", ), + path("rules/import/", login_required(OPMLImportView.as_view()), name="import"), + # Reddit path( - "rules/subreddits/create/", + "subreddits/create/", login_required(SubRedditCreateView.as_view()), name="subreddit-create", ), path( - "rules/subreddits//", + "subreddits//", login_required(SubRedditUpdateView.as_view()), name="subreddit-update", ), - path("rules/import/", login_required(OPMLImportView.as_view()), name="import"), + # Twitter + path( + "twitter/timelines/create/", + login_required(TwitterTimelineCreateView.as_view()), + name="twitter-timeline-create", + ), + path( + "twitter/timelines//", + login_required(TwitterTimelineUpdateView.as_view()), + name="twitter-timeline-update", + ), ] diff --git a/src/newsreader/news/collection/views/__init__.py b/src/newsreader/news/collection/views/__init__.py index 20769f3..c66c5a5 100644 --- a/src/newsreader/news/collection/views/__init__.py +++ b/src/newsreader/news/collection/views/__init__.py @@ -1,3 +1,8 @@ +from newsreader.news.collection.views.feed import ( + FeedCreateView, + FeedUpdateView, + OPMLImportView, +) from newsreader.news.collection.views.reddit import ( SubRedditCreateView, SubRedditUpdateView, @@ -6,8 +11,9 @@ from newsreader.news.collection.views.rules import ( CollectionRuleBulkDeleteView, CollectionRuleBulkDisableView, CollectionRuleBulkEnableView, - CollectionRuleCreateView, CollectionRuleListView, - CollectionRuleUpdateView, - OPMLImportView, +) +from newsreader.news.collection.views.twitter import ( + TwitterTimelineCreateView, + TwitterTimelineUpdateView, ) diff --git a/src/newsreader/news/collection/views/base.py b/src/newsreader/news/collection/views/base.py index e7f7b63..982a4d3 100644 --- a/src/newsreader/news/collection/views/base.py +++ b/src/newsreader/news/collection/views/base.py @@ -2,7 +2,6 @@ from django.urls import reverse_lazy import pytz -from newsreader.news.collection.forms import CollectionRuleForm from newsreader.news.collection.models import CollectionRule from newsreader.news.core.models import Category @@ -17,7 +16,6 @@ class CollectionRuleViewMixin: class CollectionRuleDetailMixin: success_url = reverse_lazy("news:collection:rules") - form_class = CollectionRuleForm def get_context_data(self, **kwargs): context_data = super().get_context_data(**kwargs) diff --git a/src/newsreader/news/collection/views/feed.py b/src/newsreader/news/collection/views/feed.py new file mode 100644 index 0000000..872d716 --- /dev/null +++ b/src/newsreader/news/collection/views/feed.py @@ -0,0 +1,62 @@ +from django.contrib import messages +from django.urls import reverse +from django.utils.translation import gettext as _ +from django.views.generic.edit import CreateView, FormView, UpdateView + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms import ( + CollectionRuleBulkForm, + FeedForm, + OPMLImportForm, +) +from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.views.base import ( + CollectionRuleDetailMixin, + CollectionRuleViewMixin, +) +from newsreader.utils.opml import parse_opml + + +class FeedUpdateView(CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView): + template_name = "news/collection/views/feed-update.html" + form_class = FeedForm + + def get_queryset(self): + queryset = super().get_queryset() + return queryset.filter(type=RuleTypeChoices.feed) + + +class FeedCreateView(CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView): + template_name = "news/collection/views/feed-create.html" + form_class = FeedForm + context_object_name = "feed" + + +class OPMLImportView(FormView): + form_class = OPMLImportForm + template_name = "news/collection/views/import.html" + + def form_valid(self, form): + user = self.request.user + file = form.cleaned_data["file"] + skip_existing = form.cleaned_data["skip_existing"] + + instances = parse_opml(file, user, skip_existing=skip_existing) + + try: + feeds = CollectionRule.objects.bulk_create(instances) + except IOError: + form.add_error("file", _("Invalid OPML file")) + return self.form_invalid(form) + + if not feeds: + form.add_error("file", _("No (new) feeds found")) + return self.form_invalid(form) + + message = _(f"{len(feeds)} new feeds created") + messages.success(self.request, message) + + return super().form_valid(form) + + def get_success_url(self): + return reverse("news:collection:rules") diff --git a/src/newsreader/news/collection/views/reddit.py b/src/newsreader/news/collection/views/reddit.py index 62ec408..4e44e3f 100644 --- a/src/newsreader/news/collection/views/reddit.py +++ b/src/newsreader/news/collection/views/reddit.py @@ -1,7 +1,7 @@ from django.views.generic.edit import CreateView, UpdateView from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.forms import SubRedditRuleForm +from newsreader.news.collection.forms import SubRedditForm from newsreader.news.collection.views.base import ( CollectionRuleDetailMixin, CollectionRuleViewMixin, @@ -11,14 +11,14 @@ from newsreader.news.collection.views.base import ( class SubRedditCreateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView ): - form_class = SubRedditRuleForm + form_class = SubRedditForm template_name = "news/collection/views/subreddit-create.html" class SubRedditUpdateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView ): - form_class = SubRedditRuleForm + form_class = SubRedditForm template_name = "news/collection/views/subreddit-update.html" context_object_name = "subreddit" diff --git a/src/newsreader/news/collection/views/rules.py b/src/newsreader/news/collection/views/rules.py index e020b67..902eedf 100644 --- a/src/newsreader/news/collection/views/rules.py +++ b/src/newsreader/news/collection/views/rules.py @@ -2,17 +2,14 @@ from django.contrib import messages from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext as _ -from django.views.generic.edit import CreateView, FormView, UpdateView +from django.views.generic.edit import FormView from django.views.generic.list import ListView -from newsreader.news.collection.choices import RuleTypeChoices -from newsreader.news.collection.forms import CollectionRuleBulkForm, OPMLImportForm -from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.forms import CollectionRuleBulkForm from newsreader.news.collection.views.base import ( CollectionRuleDetailMixin, CollectionRuleViewMixin, ) -from newsreader.utils.opml import parse_opml class CollectionRuleListView(CollectionRuleViewMixin, ListView): @@ -21,23 +18,6 @@ class CollectionRuleListView(CollectionRuleViewMixin, ListView): context_object_name = "rules" -class CollectionRuleUpdateView( - CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView -): - template_name = "news/collection/views/rule-update.html" - context_object_name = "rule" - - def get_queryset(self): - queryset = super().get_queryset() - return queryset.filter(type=RuleTypeChoices.feed) - - -class CollectionRuleCreateView( - CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView -): - template_name = "news/collection/views/rule-create.html" - - class CollectionRuleBulkView(FormView): form_class = CollectionRuleBulkForm @@ -90,33 +70,3 @@ class CollectionRuleBulkDeleteView(CollectionRuleBulkView): rule.delete() return response - - -class OPMLImportView(FormView): - form_class = OPMLImportForm - template_name = "news/collection/views/import.html" - - def form_valid(self, form): - user = self.request.user - file = form.cleaned_data["file"] - skip_existing = form.cleaned_data["skip_existing"] - - instances = parse_opml(file, user, skip_existing=skip_existing) - - try: - rules = CollectionRule.objects.bulk_create(instances) - except IOError: - form.add_error("file", _("Invalid OPML file")) - return self.form_invalid(form) - - if not rules: - form.add_error("file", _("No (new) rules found")) - return self.form_invalid(form) - - message = _(f"{len(rules)} new rules created") - messages.success(self.request, message) - - return super().form_valid(form) - - def get_success_url(self): - return reverse("news:collection:rules") diff --git a/src/newsreader/news/collection/views/twitter.py b/src/newsreader/news/collection/views/twitter.py new file mode 100644 index 0000000..db87030 --- /dev/null +++ b/src/newsreader/news/collection/views/twitter.py @@ -0,0 +1,29 @@ +from django.views.generic.edit import CreateView, UpdateView + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.forms import TwitterTimelineForm +from newsreader.news.collection.views.base import ( + CollectionRuleDetailMixin, + CollectionRuleViewMixin, +) + + +# TODO add tests +class TwitterTimelineCreateView( + CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView +): + form_class = TwitterTimelineForm + template_name = "news/collection/views/twitter/timeline-create.html" + + +# TODO add tests +class TwitterTimelineUpdateView( + CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView +): + form_class = TwitterTimelineForm + template_name = "news/collection/views/twitter/timeline-update.html" + context_object_name = "timeline" + + def get_queryset(self): + queryset = super().get_queryset() + return queryset.filter(type=RuleTypeChoices.twitter_timeline) diff --git a/src/newsreader/utils/opml.py b/src/newsreader/utils/opml.py index 55a9387..1aca0fd 100644 --- a/src/newsreader/utils/opml.py +++ b/src/newsreader/utils/opml.py @@ -38,4 +38,5 @@ def parse_opml(file, user, skip_existing=False): logging.info(f"Skipped due to invalid URL: {e}") continue + # TODO create feed type rules yield CollectionRule(url=feed_url, name=name, user=user) From 431827986e978c3f62c33eeb0eae80160d7c4759 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 20 Sep 2020 20:36:10 +0200 Subject: [PATCH 36/69] Passthrough urls from backend --- src/newsreader/js/pages/categories/App.js | 3 +- .../categories/components/CategoryCard.js | 2 +- src/newsreader/js/pages/categories/index.js | 12 ++++++- src/newsreader/js/pages/homepage/App.js | 10 +++++- .../js/pages/homepage/components/PostModal.js | 23 +++++++++---- .../homepage/components/postlist/PostItem.js | 20 ++++++++--- .../homepage/components/postlist/PostList.js | 11 +++++- src/newsreader/js/pages/homepage/index.js | 12 ++++++- .../templates/news/core/views/categories.html | 3 ++ .../templates/news/core/views/homepage.html | 11 +++++- src/newsreader/news/core/views.py | 34 ++++++++++++------- 11 files changed, 109 insertions(+), 32 deletions(-) diff --git a/src/newsreader/js/pages/categories/App.js b/src/newsreader/js/pages/categories/App.js index 691aaed..a035b46 100644 --- a/src/newsreader/js/pages/categories/App.js +++ b/src/newsreader/js/pages/categories/App.js @@ -69,6 +69,7 @@ class App extends React.Component { key={category.pk} category={category} showDialog={this.selectCategory} + updateUrl={this.props.updateUrl} /> ); }); @@ -80,7 +81,7 @@ class App extends React.Component { const pageHeader = ( <>

Categories

- + Create category diff --git a/src/newsreader/js/pages/categories/components/CategoryCard.js b/src/newsreader/js/pages/categories/components/CategoryCard.js index 94bd6f4..2e7cad4 100644 --- a/src/newsreader/js/pages/categories/components/CategoryCard.js +++ b/src/newsreader/js/pages/categories/components/CategoryCard.js @@ -33,7 +33,7 @@ const CategoryCard = props => { <> Edit diff --git a/src/newsreader/js/pages/categories/index.js b/src/newsreader/js/pages/categories/index.js index 9d75bb9..791fdbd 100644 --- a/src/newsreader/js/pages/categories/index.js +++ b/src/newsreader/js/pages/categories/index.js @@ -9,5 +9,15 @@ if (page) { const dataScript = document.getElementById('categories-data'); const categories = JSON.parse(dataScript.textContent); - ReactDOM.render(, page); + let createUrl = document.getElementById('createUrl').textContent; + let updateUrl = document.getElementById('updateUrl').textContent; + + ReactDOM.render( + , + page + ); } diff --git a/src/newsreader/js/pages/homepage/App.js b/src/newsreader/js/pages/homepage/App.js index 91cfa4e..77b6222 100644 --- a/src/newsreader/js/pages/homepage/App.js +++ b/src/newsreader/js/pages/homepage/App.js @@ -19,7 +19,11 @@ class App extends React.Component { return ( <> - + {this.props.error && ( @@ -30,6 +34,10 @@ class App extends React.Component { post={this.props.post} rule={this.props.rule} category={this.props.category} + feedUrl={this.props.feedUrl} + subredditUrl={this.props.subredditUrl} + timelineUrl={this.props.timelineUrl} + categoriesUrl={this.props.categoriesUrl} /> )} diff --git a/src/newsreader/js/pages/homepage/components/PostModal.js b/src/newsreader/js/pages/homepage/components/PostModal.js index 3d7a888..5196102 100644 --- a/src/newsreader/js/pages/homepage/components/PostModal.js +++ b/src/newsreader/js/pages/homepage/components/PostModal.js @@ -3,7 +3,13 @@ import { connect } from 'react-redux'; import Cookies from 'js-cookie'; import { unSelectPost, markPostRead } from '../actions/posts.js'; -import { CATEGORY_TYPE, RULE_TYPE, FEED, SUBREDDIT } from '../constants.js'; +import { + CATEGORY_TYPE, + RULE_TYPE, + FEED, + SUBREDDIT, + TWITTER_TIMELINE, +} from '../constants.js'; import { formatDatetime } from '../../../utils.js'; class PostModal extends React.Component { @@ -44,12 +50,15 @@ class PostModal extends React.Component { const post = this.props.post; const publicationDate = formatDatetime(post.publicationDate); const titleClassName = post.read ? 'post__title post__title--read' : 'post__title'; + let ruleUrl = ''; - // TODO add mapping & get urls from backend - const ruleUrl = - this.props.rule.type === FEED - ? `/collection/rules/${this.props.rule.id}/` - : `/collection/rules/subreddits/${this.props.rule.id}/`; + if (this.props.rule.type === SUBREDDIT) { + ruleUrl = `${this.props.subredditUrl}/${this.props.rule.id}/`; + } else if (this.props.rule.type === TWITTER_TIMELINE) { + ruleUrl = `${this.props.timelineUrl}/${this.props.rule.id}/`; + } else { + ruleUrl = `${this.props.feedUrl}/${this.props.rule.id}/`; + } return (
@@ -68,7 +77,7 @@ class PostModal extends React.Component { {this.props.category && ( diff --git a/src/newsreader/js/pages/homepage/components/postlist/PostItem.js b/src/newsreader/js/pages/homepage/components/postlist/PostItem.js index 9b64289..f69a463 100644 --- a/src/newsreader/js/pages/homepage/components/postlist/PostItem.js +++ b/src/newsreader/js/pages/homepage/components/postlist/PostItem.js @@ -1,7 +1,13 @@ import React from 'react'; import { connect } from 'react-redux'; -import { CATEGORY_TYPE, RULE_TYPE, FEED, SUBREDDIT } from '../../constants.js'; +import { + CATEGORY_TYPE, + RULE_TYPE, + FEED, + SUBREDDIT, + TWITTER_TIMELINE, +} from '../../constants.js'; import { selectPost } from '../../actions/posts.js'; import { formatDatetime } from '../../../../utils.js'; @@ -13,11 +19,15 @@ class PostItem extends React.Component { const titleClassName = post.read ? 'posts__header posts__header--read' : 'posts__header'; + let ruleUrl = ''; - const ruleUrl = - rule.type === FEED - ? `/collection/rules/${rule.id}/` - : `/collection/rules/subreddits/${rule.id}/`; + if (rule.type === SUBREDDIT) { + ruleUrl = `${this.props.subredditUrl}/${rule.id}/`; + } else if (rule.type === TWITTER_TIMELINE) { + ruleUrl = `${this.props.timelineUrl}/${rule.id}/`; + } else { + ruleUrl = `${this.props.feedUrl}/${rule.id}/`; + } return (
  • diff --git a/src/newsreader/js/pages/homepage/components/postlist/PostList.js b/src/newsreader/js/pages/homepage/components/postlist/PostList.js index cd57d6d..cff2437 100644 --- a/src/newsreader/js/pages/homepage/components/postlist/PostList.js +++ b/src/newsreader/js/pages/homepage/components/postlist/PostList.js @@ -38,7 +38,16 @@ class PostList extends React.Component { render() { const postItems = this.props.postsBySection.map((item, index) => { - return ; + return ( + + ); }); if (isEqual(this.props.selected, {})) { diff --git a/src/newsreader/js/pages/homepage/index.js b/src/newsreader/js/pages/homepage/index.js index c16ed39..394a06c 100644 --- a/src/newsreader/js/pages/homepage/index.js +++ b/src/newsreader/js/pages/homepage/index.js @@ -11,9 +11,19 @@ const page = document.getElementById('homepage--page'); if (page) { const store = configureStore(); + let feedUrl = document.getElementById('feedUrl').textContent; + let subredditUrl = document.getElementById('subredditUrl').textContent; + let timelineUrl = document.getElementById('timelineUrl').textContent; + let categoriesUrl = document.getElementById('categoriesUrl').textContent; + ReactDOM.render( - + , page ); diff --git a/src/newsreader/news/core/templates/news/core/views/categories.html b/src/newsreader/news/core/templates/news/core/views/categories.html index 35fc741..6a6cdae 100644 --- a/src/newsreader/news/core/templates/news/core/views/categories.html +++ b/src/newsreader/news/core/templates/news/core/views/categories.html @@ -30,5 +30,8 @@ ] + {{ categories_update_url|json_script:"updateUrl" }} + {{ categories_create_url|json_script:"createUrl" }} + {{ block.super }} {% endblock %} diff --git a/src/newsreader/news/core/templates/news/core/views/homepage.html b/src/newsreader/news/core/templates/news/core/views/homepage.html index 79e1ccc..502ef63 100644 --- a/src/newsreader/news/core/templates/news/core/views/homepage.html +++ b/src/newsreader/news/core/templates/news/core/views/homepage.html @@ -3,4 +3,13 @@ {% block content %}
    -{% endblock %} +{% endblock content %} + +{% block scripts %} + {{ feed_url|json_script:"feedUrl" }} + {{ subreddit_url|json_script:"subredditUrl" }} + {{ twitter_timeline_url|json_script:"timelineUrl" }} + {{ categories_url|json_script:"categoriesUrl" }} + + {{ block.super }} +{% endblock scripts %} diff --git a/src/newsreader/news/core/views.py b/src/newsreader/news/core/views.py index 9ef81eb..981e7b2 100644 --- a/src/newsreader/news/core/views.py +++ b/src/newsreader/news/core/views.py @@ -11,24 +11,21 @@ from newsreader.news.core.models import Category class NewsView(TemplateView): template_name = "news/core/views/homepage.html" - # TODO serialize objects to show filled main page def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - user = self.request.user - categories = { - category: category.rules.order_by("-created") - for category in user.categories.order_by("name") + return { + **context, + "feed_url": reverse_lazy("news:collection:feed-update", args=(0,)), + "subreddit_url": reverse_lazy( + "news:collection:subreddit-update", args=(0,) + ), + "twitter_timeline_url": reverse_lazy( + "news:collection:twitter-timeline-update", args=(0,) + ), + "categories_url": reverse_lazy("news:core:category-update", args=(0,)), } - rules = { - rule: rule.posts.order_by("-publication_date")[:30] - for rule in user.rules.order_by("-created") - } - - context.update(categories=categories, rules=rules) - return context - class CategoryViewMixin: queryset = Category.objects.prefetch_related("rules").order_by("name") @@ -58,6 +55,17 @@ class CategoryListView(CategoryViewMixin, ListView): template_name = "news/core/views/categories.html" context_object_name = "categories" + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + + return { + **context, + "categories_create_url": reverse_lazy("news:core:category-create"), + "categories_update_url": ( + reverse_lazy("news:core:category-update", args=(0,)) + ), + } + class CategoryUpdateView(CategoryViewMixin, CategoryDetailMixin, UpdateView): template_name = "news/core/views/category-update.html" From a3deafb610e437dc2d50ce9f936bfb7e579ead28 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 20 Sep 2020 22:16:38 +0200 Subject: [PATCH 37/69] Add todo --- src/newsreader/news/collection/reddit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 1cb9e29..284c82f 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -243,6 +243,7 @@ class RedditStream(PostStream): ) from e +# TODO fix client class RedditClient(PostClient): stream = RedditStream From f3b4319c2d6512def55d7aad442598171c870a97 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 21 Sep 2020 21:52:27 +0200 Subject: [PATCH 38/69] Fix reddit client --- src/newsreader/news/collection/base.py | 13 ++----------- src/newsreader/news/collection/reddit.py | 1 - 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/newsreader/news/collection/base.py b/src/newsreader/news/collection/base.py index eb11619..7286526 100644 --- a/src/newsreader/news/collection/base.py +++ b/src/newsreader/news/collection/base.py @@ -4,7 +4,6 @@ from newsreader.news.collection.constants import ( WHITELISTED_ATTRIBUTES, WHITELISTED_TAGS, ) -from newsreader.news.collection.models import CollectionRule from newsreader.news.core.models import Post @@ -33,7 +32,7 @@ class Client: stream = Stream def __init__(self, rules=[]): - self.rules = rules if rules else CollectionRule.objects.enabled() + self.rules = rules def __enter__(self): for rule in self.rules: @@ -130,14 +129,6 @@ class PostStream(Stream): class PostClient(Client): stream = PostStream - def __init__(self, rules=[]): - if rules: - self.rules = rules - else: - self.rules = CollectionRule.objects.enabled().filter( - type=self.stream.rule_type - ) - def set_rule_error(self, rule, exception): length = rule._meta.get_field("error").max_length @@ -146,7 +137,7 @@ class PostClient(Client): class PostCollector(Collector): - def collect(self, rules=None): + def collect(self, rules=[]): with self.client(rules=rules) as client: for payload, stream in client: with self.builder(payload, stream) as builder: diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 284c82f..1cb9e29 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -243,7 +243,6 @@ class RedditStream(PostStream): ) from e -# TODO fix client class RedditClient(PostClient): stream = RedditStream From c6fda4fceeb5b5a106483d4f1044a19646b06cd7 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 21 Sep 2020 22:21:08 +0200 Subject: [PATCH 39/69] Don't show initially created rules as failed --- src/newsreader/news/collection/models.py | 4 ++++ .../collection/templates/news/collection/views/rules.html | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/newsreader/news/collection/models.py b/src/newsreader/news/collection/models.py index d1d8024..23d76f6 100644 --- a/src/newsreader/news/collection/models.py +++ b/src/newsreader/news/collection/models.py @@ -71,3 +71,7 @@ class CollectionRule(TimeStampedModel): return reverse("news:collection:subreddit-update", kwargs={"pk": self.pk}) return reverse("news:collection:feed-update", kwargs={"pk": self.pk}) + + @property + def failed(self): + return not self.succeeded and self.last_run diff --git a/src/newsreader/news/collection/templates/news/collection/views/rules.html b/src/newsreader/news/collection/templates/news/collection/views/rules.html index cd98b29..678716e 100644 --- a/src/newsreader/news/collection/templates/news/collection/views/rules.html +++ b/src/newsreader/news/collection/templates/news/collection/views/rules.html @@ -37,7 +37,7 @@ {% for rule in rules %} - + {% with rule|id_for_label:"rules" as id_for_label %} {% include "components/form/checkbox.html" with name="rules" value=rule.pk id=id_for_label id_for_label=id_for_label %} @@ -55,10 +55,10 @@     {{ rule.url }} - {% if rule.succeeded %} - - {% else %} + {% if rule.failed %} + {% else %} + {% endif %} From fed811a2272889d37771304a9e7584d54f24caef Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 21 Sep 2020 22:22:29 +0200 Subject: [PATCH 40/69] Add twitter time line url to property --- src/newsreader/news/collection/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/models.py b/src/newsreader/news/collection/models.py index 23d76f6..92dfe51 100644 --- a/src/newsreader/news/collection/models.py +++ b/src/newsreader/news/collection/models.py @@ -64,11 +64,14 @@ class CollectionRule(TimeStampedModel): def __str__(self): return self.name - # TODO add twitter url @property def update_url(self): if self.type == RuleTypeChoices.subreddit: return reverse("news:collection:subreddit-update", kwargs={"pk": self.pk}) + elif self.type == RuleTypeChoices.twitter_timeline: + return reverse( + "news:collection:twitter-timeline-update", kwargs={"pk": self.pk} + ) return reverse("news:collection:feed-update", kwargs={"pk": self.pk}) From 3e3af4ce95ae1c19dcef1849365c054873bef839 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 21 Sep 2020 22:44:49 +0200 Subject: [PATCH 41/69] Add twitter timeline create tests --- .../news/collection/forms/twitter.py | 2 ++ .../news/collection/tests/factories.py | 2 +- .../tests/views/test_twitter_views.py | 36 +++++++++++++++++++ .../news/collection/views/twitter.py | 1 - 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 src/newsreader/news/collection/tests/views/test_twitter_views.py diff --git a/src/newsreader/news/collection/forms/twitter.py b/src/newsreader/news/collection/forms/twitter.py index 441b243..d856fdc 100644 --- a/src/newsreader/news/collection/forms/twitter.py +++ b/src/newsreader/news/collection/forms/twitter.py @@ -6,6 +6,7 @@ import pytz from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.forms.base import CollectionRuleForm from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.twitter import TWITTER_URL class TwitterTimelineForm(CollectionRuleForm): @@ -20,6 +21,7 @@ class TwitterTimelineForm(CollectionRuleForm): instance.type = RuleTypeChoices.twitter_timeline instance.timezone = str(pytz.utc) + instance.url = f"{TWITTER_URL}/{instance.screen_name}" if commit: instance.save() diff --git a/src/newsreader/news/collection/tests/factories.py b/src/newsreader/news/collection/tests/factories.py index 81fa8ac..26f66cc 100644 --- a/src/newsreader/news/collection/tests/factories.py +++ b/src/newsreader/news/collection/tests/factories.py @@ -30,6 +30,6 @@ class SubredditFactory(CollectionRuleFactory): website_url = REDDIT_URL -class TwitterTimeLineFactory(CollectionRuleFactory): +class TwitterTimelineFactory(CollectionRuleFactory): type = RuleTypeChoices.twitter_timeline screen_name = factory.Faker("user_name") diff --git a/src/newsreader/news/collection/tests/views/test_twitter_views.py b/src/newsreader/news/collection/tests/views/test_twitter_views.py new file mode 100644 index 0000000..bae5756 --- /dev/null +++ b/src/newsreader/news/collection/tests/views/test_twitter_views.py @@ -0,0 +1,36 @@ +from django.test import TestCase +from django.urls import reverse + +import pytz + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCase +from newsreader.news.collection.twitter import TWITTER_URL + + +class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): + def setUp(self): + super().setUp() + + self.form_data = { + "name": "new rule", + "screen_name": "RobertsSpaceInd", + "category": str(self.category.pk), + } + + self.url = reverse("news:collection:twitter-timeline-create") + + def test_creation(self): + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 302) + + rule = CollectionRule.objects.get(name="new rule") + + self.assertEquals(rule.type, RuleTypeChoices.twitter_timeline) + self.assertEquals(rule.url, f"{TWITTER_URL}/RobertsSpaceInd") + self.assertEquals(rule.timezone, str(pytz.utc)) + self.assertEquals(rule.favicon, None) + self.assertEquals(rule.category.pk, self.category.pk) + self.assertEquals(rule.user.pk, self.user.pk) diff --git a/src/newsreader/news/collection/views/twitter.py b/src/newsreader/news/collection/views/twitter.py index db87030..d854740 100644 --- a/src/newsreader/news/collection/views/twitter.py +++ b/src/newsreader/news/collection/views/twitter.py @@ -8,7 +8,6 @@ from newsreader.news.collection.views.base import ( ) -# TODO add tests class TwitterTimelineCreateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView ): From 0411d7659650105d8cd7f0bc1afcaab7627ee381 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 21 Sep 2020 22:56:07 +0200 Subject: [PATCH 42/69] Add twitter timeline update view tests --- .../news/collection/forms/twitter.py | 1 + .../tests/views/test_twitter_views.py | 78 +++++++++++++++++++ .../news/collection/views/twitter.py | 1 - 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/forms/twitter.py b/src/newsreader/news/collection/forms/twitter.py index d856fdc..5603c7a 100644 --- a/src/newsreader/news/collection/forms/twitter.py +++ b/src/newsreader/news/collection/forms/twitter.py @@ -14,6 +14,7 @@ class TwitterTimelineForm(CollectionRuleForm): max_length=255, label=_("Twitter profile name"), help_text=_("Profile name without hashtags"), + required=True, ) def save(self, commit=True): diff --git a/src/newsreader/news/collection/tests/views/test_twitter_views.py b/src/newsreader/news/collection/tests/views/test_twitter_views.py index bae5756..4f178cc 100644 --- a/src/newsreader/news/collection/tests/views/test_twitter_views.py +++ b/src/newsreader/news/collection/tests/views/test_twitter_views.py @@ -1,3 +1,4 @@ + from django.test import TestCase from django.urls import reverse @@ -5,8 +6,10 @@ import pytz from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.models import CollectionRule +from newsreader.news.collection.tests.factories import TwitterTimelineFactory from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCase from newsreader.news.collection.twitter import TWITTER_URL +from newsreader.news.core.tests.factories import CategoryFactory class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): @@ -34,3 +37,78 @@ class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): self.assertEquals(rule.favicon, None) self.assertEquals(rule.category.pk, self.category.pk) self.assertEquals(rule.user.pk, self.user.pk) + + +class TwitterTimelineUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): + def setUp(self): + super().setUp() + + self.rule = TwitterTimelineFactory( + name="Star citizen", + screen_name="RobertsSpaceInd", + user=self.user, + category=self.category, + type=RuleTypeChoices.twitter_timeline, + ) + self.url = reverse( + "news:collection:twitter-timeline-update", kwargs={"pk": self.rule.pk} + ) + + self.form_data = { + "name": self.rule.name, + "screen_name": self.rule.screen_name, + "category": str(self.category.pk), + "timezone": pytz.utc, + } + + def test_name_change(self): + self.form_data.update(name="Star citizen Twitter") + + response = self.client.post(self.url, self.form_data) + self.assertEquals(response.status_code, 302) + + self.rule.refresh_from_db() + + self.assertEquals(self.rule.name, "Star citizen Twitter") + + def test_category_change(self): + new_category = CategoryFactory(user=self.user) + + self.form_data.update(category=new_category.pk) + + response = self.client.post(self.url, self.form_data) + self.assertEquals(response.status_code, 302) + + self.rule.refresh_from_db() + + self.assertEquals(self.rule.category.pk, new_category.pk) + + def test_twitter_timelines_only(self): + rule = TwitterTimelineFactory( + name="Fake twitter", + user=self.user, + category=self.category, + type=RuleTypeChoices.feed, + url="https://twitter.com/RobertsSpaceInd", + ) + url = reverse("news:collection:twitter-timeline-update", kwargs={"pk": rule.pk}) + + response = self.client.get(url) + + self.assertEquals(response.status_code, 404) + + def test_screen_name_change(self): + self.form_data.update(screen_name="CyberpunkGame") + + response = self.client.post(self.url, self.form_data) + + self.assertEquals(response.status_code, 302) + + self.rule.refresh_from_db() + + self.assertEquals(self.rule.type, RuleTypeChoices.twitter_timeline) + self.assertEquals(self.rule.url, f"{TWITTER_URL}/CyberpunkGame") + self.assertEquals(self.rule.timezone, str(pytz.utc)) + self.assertEquals(self.rule.favicon, None) + self.assertEquals(self.rule.category.pk, self.category.pk) + self.assertEquals(self.rule.user.pk, self.user.pk) diff --git a/src/newsreader/news/collection/views/twitter.py b/src/newsreader/news/collection/views/twitter.py index d854740..c60dbda 100644 --- a/src/newsreader/news/collection/views/twitter.py +++ b/src/newsreader/news/collection/views/twitter.py @@ -15,7 +15,6 @@ class TwitterTimelineCreateView( template_name = "news/collection/views/twitter/timeline-create.html" -# TODO add tests class TwitterTimelineUpdateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView ): From b653c210f14a5a3900b85889834230e6fb222309 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 21 Sep 2020 23:15:04 +0200 Subject: [PATCH 43/69] Fix tests --- .../collection/tests/feed/collector/tests.py | 44 ++++++------------- .../collection/tests/twitter/builder/tests.py | 22 +++++----- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/src/newsreader/news/collection/tests/feed/collector/tests.py b/src/newsreader/news/collection/tests/feed/collector/tests.py index fea10bd..a7f3573 100644 --- a/src/newsreader/news/collection/tests/feed/collector/tests.py +++ b/src/newsreader/news/collection/tests/feed/collector/tests.py @@ -42,10 +42,10 @@ class FeedCollectorTestCase(TestCase): def test_simple_batch(self): self.mocked_parse.return_value = multiple_mock - rule = FeedFactory() + rule = FeedFactory() collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -57,10 +57,11 @@ class FeedCollectorTestCase(TestCase): def test_emtpy_batch(self): self.mocked_fetch.return_value = Mock() self.mocked_parse.return_value = empty_mock + rule = FeedFactory() collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -71,10 +72,10 @@ class FeedCollectorTestCase(TestCase): def test_not_found(self): self.mocked_fetch.side_effect = StreamNotFoundException - rule = FeedFactory() + rule = FeedFactory() collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -84,11 +85,12 @@ class FeedCollectorTestCase(TestCase): def test_denied(self): self.mocked_fetch.side_effect = StreamDeniedException + old_run = timezone.make_aware(datetime(2019, 10, 30, 12, 30)) rule = FeedFactory(last_run=old_run) collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -99,11 +101,12 @@ class FeedCollectorTestCase(TestCase): def test_forbidden(self): self.mocked_fetch.side_effect = StreamForbiddenException + old_run = pytz.utc.localize(datetime(2019, 10, 30, 12, 30)) rule = FeedFactory(last_run=old_run) collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -114,13 +117,14 @@ class FeedCollectorTestCase(TestCase): def test_timed_out(self): self.mocked_fetch.side_effect = StreamTimeOutException + last_run = timezone.make_aware( datetime.combine(date=date(2019, 10, 30), time=time(12, 30)) ) rule = FeedFactory(last_run=last_run) collector = FeedCollector() - collector.collect() + collector.collect(rules=[rule]) rule.refresh_from_db() @@ -133,6 +137,7 @@ class FeedCollectorTestCase(TestCase): def test_duplicates(self): self.mocked_parse.return_value = duplicate_mock + rule = FeedFactory() aware_datetime = build_publication_date( @@ -240,26 +245,3 @@ class FeedCollectorTestCase(TestCase): self.assertEquals( third_post.title, "Birmingham head teacher threatened over LGBT lessons" ) - - def test_disabled_rules(self): - old_run = pytz.utc.localize(datetime(2019, 10, 28, 15)) - rules = ( - FeedFactory(enabled=False, last_run=old_run), - FeedFactory(enabled=True, last_run=old_run), - ) - - self.mocked_parse.return_value = multiple_mock - - collector = FeedCollector() - collector.collect() - - for rule in rules: - rule.refresh_from_db() - - self.assertEquals(Post.objects.count(), 3) - self.assertEquals(rules[1].succeeded, True) - self.assertEquals(rules[1].last_run, timezone.now()) - self.assertEquals(rules[1].error, None) - - self.assertEquals(rules[0].last_run, old_run) - self.assertEquals(rules[0].succeeded, False) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 2527fb9..2943f48 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -8,7 +8,7 @@ import pytz from ftfy import fix_text -from newsreader.news.collection.tests.factories import TwitterTimeLineFactory +from newsreader.news.collection.tests.factories import TwitterTimelineFactory from newsreader.news.collection.tests.twitter.builder.mocks import ( gif_mock, image_mock, @@ -31,7 +31,7 @@ class TwitterBuilderTestCase(TestCase): def test_simple_post(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(simple_mock, mock_stream) as builder: @@ -91,7 +91,7 @@ class TwitterBuilderTestCase(TestCase): def test_images_in_post(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(image_mock, mock_stream) as builder: @@ -134,7 +134,7 @@ class TwitterBuilderTestCase(TestCase): def test_videos_in_post(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(video_mock, mock_stream) as builder: @@ -190,7 +190,7 @@ class TwitterBuilderTestCase(TestCase): def test_video_without_bitrate(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(video_without_bitrate_mock, mock_stream) as builder: @@ -212,7 +212,7 @@ class TwitterBuilderTestCase(TestCase): def test_GIFs_in_post(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(gif_mock, mock_stream) as builder: @@ -241,7 +241,7 @@ class TwitterBuilderTestCase(TestCase): def test_retweet_post(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(retweet_mock, mock_stream) as builder: @@ -278,7 +278,7 @@ class TwitterBuilderTestCase(TestCase): def test_quoted_post(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(quoted_mock, mock_stream) as builder: @@ -312,7 +312,7 @@ class TwitterBuilderTestCase(TestCase): def test_empty_data(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder([], mock_stream) as builder: @@ -324,7 +324,7 @@ class TwitterBuilderTestCase(TestCase): def test_html_sanitizing(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(unsanitized_mock, mock_stream) as builder: @@ -364,7 +364,7 @@ class TwitterBuilderTestCase(TestCase): def test_urlize_on_urls(self): builder = TwitterBuilder - profile = TwitterTimeLineFactory(screen_name="RobertsSpaceInd") + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") mock_stream = Mock(rule=profile) with builder(simple_mock, mock_stream) as builder: From 5618ca702dc5339d1a4781214e6e3a9625080633 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Wed, 23 Sep 2020 23:21:00 +0200 Subject: [PATCH 44/69] Add requests-oauthlib to deps --- poetry.lock | 39 ++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 9dd9d1b..0bbd4e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -544,6 +544,19 @@ category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +[[package]] +name = "oauthlib" +version = "3.1.0" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +rsa = ["cryptography"] +signals = ["blinker"] +signedtoken = ["cryptography", "pyjwt (>=1.0.0)"] + [[package]] name = "packaging" version = "20.3" @@ -654,6 +667,21 @@ chardet = ">=3.0.2,<4" idna = ">=2.5,<3" urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" +[[package]] +name = "requests-oauthlib" +version = "1.3.0" +description = "OAuthlib authentication support for Requests." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +rsa = ["oauthlib (>=3.0.0)"] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + [[package]] name = "ruamel.yaml" version = "0.16.10" @@ -815,7 +843,7 @@ testing = ["jaraco.itertools", "func-timeout"] [metadata] lock-version = "1.0" python-versions = "^3.7" -content-hash = "c1083804e80713b99e49109f59560ebfaeba67c4a903a15130a9748c23922b1a" +content-hash = "cda651cbf92ffc53c6ef09bea6204f5927b5a1bf3feff85bc70fa672e526cc91" [metadata.files] amqp = [ @@ -1069,6 +1097,10 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] +oauthlib = [ + {file = "oauthlib-3.1.0-py2.py3-none-any.whl", hash = "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea"}, + {file = "oauthlib-3.1.0.tar.gz", hash = "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889"}, +] packaging = [ {file = "packaging-20.3-py2.py3-none-any.whl", hash = "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752"}, {file = "packaging-20.3.tar.gz", hash = "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3"}, @@ -1137,6 +1169,11 @@ requests = [ {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, ] +requests-oauthlib = [ + {file = "requests-oauthlib-1.3.0.tar.gz", hash = "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a"}, + {file = "requests_oauthlib-1.3.0-py2.py3-none-any.whl", hash = "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d"}, + {file = "requests_oauthlib-1.3.0-py3.7.egg", hash = "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"}, +] "ruamel.yaml" = [ {file = "ruamel.yaml-0.16.10-py2.py3-none-any.whl", hash = "sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b"}, {file = "ruamel.yaml-0.16.10.tar.gz", hash = "sha256:099c644a778bf72ffa00524f78dd0b6476bca94a1da344130f4bf3381ce5b954"}, diff --git a/pyproject.toml b/pyproject.toml index b235671..2d400ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ python-dotenv = "^0.12.0" django = ">=3.0.7" sentry-sdk = "^0.15.1" ftfy = "^5.8" +requests_oauthlib = "^1.3.0" [tool.poetry.dev-dependencies] factory-boy = "^2.12.0" From b498354cb64f91b5c35c659a95566fe87848d1a1 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Wed, 23 Sep 2020 23:21:55 +0200 Subject: [PATCH 45/69] Remove Twitter refresh token button Twitter does not have this feature, see https://developer.twitter.com/en/docs/authentication/faq --- .../accounts/templates/accounts/views/integrations.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html index 81a1fa1..ce565ec 100644 --- a/src/newsreader/accounts/templates/accounts/views/integrations.html +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -48,10 +48,6 @@ {% trans "Authorize account" %} - - From 52d8c14abe92a6d40d86ada91f6912eaefc0abdf Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Thu, 24 Sep 2020 21:14:08 +0200 Subject: [PATCH 46/69] Add initial twitter authorization views --- .../templates/accounts/views/twitter.html | 20 +++ src/newsreader/accounts/views/integrations.py | 122 ++++++++++++++++++ src/newsreader/news/collection/twitter.py | 3 + 3 files changed, 145 insertions(+) create mode 100644 src/newsreader/accounts/templates/accounts/views/twitter.html diff --git a/src/newsreader/accounts/templates/accounts/views/twitter.html b/src/newsreader/accounts/templates/accounts/views/twitter.html new file mode 100644 index 0000000..e2c51aa --- /dev/null +++ b/src/newsreader/accounts/templates/accounts/views/twitter.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block content %} +
    +
    + {% if error %} +

    {% trans "Twitter authorization failed" %}

    +

    {{ error }}

    + {% elif authorized %} +

    {% trans "Twitter account is linked" %}

    +

    {% trans "Your Twitter account was successfully linked." %}

    + {% endif %} + +

    + {% trans "Return to integrations page" %} +

    +
    +
    +{% endblock %} diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index fa343a2..3b30ba3 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -1,11 +1,17 @@ import logging +from urllib.parse import parse_qs, urlencode + +from django.conf import settings from django.contrib import messages from django.core.cache import cache +from django.shortcuts import redirect from django.urls import reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import RedirectView, TemplateView +from requests_oauthlib import OAuth1 as OAuth + from newsreader.news.collection.exceptions import StreamException from newsreader.news.collection.reddit import ( get_reddit_access_token, @@ -13,6 +19,12 @@ from newsreader.news.collection.reddit import ( revoke_reddit_token, ) from newsreader.news.collection.tasks import RedditTokenTask +from newsreader.news.collection.twitter import ( + TWITTER_ACCESS_TOKEN_URL, + TWITTER_AUTH_URL, + TWITTER_REQUEST_TOKEN_URL, +) +from newsreader.news.collection.utils import post logger = logging.getLogger(__name__) @@ -148,3 +160,113 @@ class RedditRevokeRedirectView(RedirectView): messages.success(request, _("Reddit account deathorized")) return response + + +# TODO hookup url to urlconf +# TODO hookup url to integrations button +class TwitterAuthRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + callback_uri=settings.TWITTER_REDIRECT_URL, + ) + + try: + response = post(TWITTER_REQUEST_TOKEN_URL, auth=oauth) + except StreamException: + logger.exception("Failed requesting Twitter authentication token") + + messages.error(request, _("Unable to retrieve initial Twitter token")) + return super().get(request, *args, **kwargs) + + params = parse_qs(response.content) + + request_oauth_token = params.get("oauth_token")[0] + request_oauth_secret = params.get("oauth_token_secret")[0] + + cache.set_many( + { + f"twitter-{request.user.email}-token": request_oauth_token, + f"twitter-{request.user.email}-secret": request_oauth_secret, + } + ) + + request_params = urlencode({"oauth_token": request_oauth_secret}) + return redirect(f"{TWITTER_AUTH_URL}/?{request_params}") + + +# TODO hookup url +class TwitterTemplateView(TemplateView): + template_name = "accounts/views/twitter.html" + + def get(self, request, *args, **kwargs): + context = self.get_context_data(**kwargs) + + denied = request.GET.get("denied", True) + oauth_token = request.GET.get("oauth_token") + request.GET.get("oauth_verifier") + + if denied: + return self.render_to_response( + { + **context, + "error": "Twitter authorization failed", + "authorized": False, + } + ) + + cached_token = cache.get(f"twitter-{request.user.email}-token") + + if oauth_token != cached_token: + return self.render_to_response( + { + **context, + "error": "OAuth tokens failed to match", + "authorized": False, + } + ) + + cached_secret = cache.get(f"twitter-{request.user.email}-secret") + + if not cached_token or not cached_token_secret: + return self.render_to_response( + { + **context, + "error": "No matching tokens found for this user", + "authorized": False, + } + ) + + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=cached_token, + resource_owner_secret=cached_secret, + verifier=oauth_token_secret, + ) + + try: + response = post(TWITTER_ACCESS_TOKEN_URL, auth=oauth) + except StreamException: + logger.exception("Failed requesting Twitter access token") + + return self.render_to_response( + { + **context, + "error": "Failed requesting access token", + "authorized": False, + } + ) + + params = parse_qs(response.content) + oauth_token = params.get("oauth_token")[0] + oauth_secret = params.get("oauth_token_secret")[0] + + request.user.twitter_oauth_token = oauth_token + request.user.twitter_oauth_token_secret = oauth_secret + request.user.save() + + return self.render_to_response({**context, "error": None, "authorized": True}) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index c2d693f..7f075d7 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -31,6 +31,9 @@ logger = logging.getLogger(__name__) TWITTER_URL = "https://twitter.com" TWITTER_API_URL = "https://api.twitter.com/1.1" +TWITTER_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" +TWITTER_AUTH_URL = "https://api.twitter.com/oauth/authorize" +TWITTER_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" class TwitterBuilder(PostBuilder): From f21b626802eb0816752f55996fbd5d8417e96115 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Thu, 24 Sep 2020 22:45:22 +0200 Subject: [PATCH 47/69] Fix views & update admin --- src/newsreader/accounts/admin.py | 18 ++++++++-- .../accounts/views/integrations.html | 24 ++++++++++---- src/newsreader/accounts/urls.py | 12 +++++++ src/newsreader/accounts/views/__init__.py | 2 ++ src/newsreader/accounts/views/integrations.py | 33 ++++++++++++------- 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/newsreader/accounts/admin.py b/src/newsreader/accounts/admin.py index 49390c7..02d372c 100644 --- a/src/newsreader/accounts/admin.py +++ b/src/newsreader/accounts/admin.py @@ -11,8 +11,18 @@ class UserAdminForm(UserChangeForm): class Meta: widgets = { "email": forms.EmailInput(attrs={"size": "50"}), - "reddit_access_token": forms.TextInput(attrs={"size": "90"}), - "reddit_refresh_token": forms.TextInput(attrs={"size": "90"}), + "reddit_access_token": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), + "reddit_refresh_token": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), + "twitter_oauth_token": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), + "twitter_oauth_token_secret": forms.PasswordInput( + attrs={"size": "90"}, render_value=True + ), } @@ -34,6 +44,10 @@ class UserAdmin(DjangoUserAdmin): _("Reddit settings"), {"fields": ("reddit_access_token", "reddit_refresh_token")}, ), + ( + _("Twitter settings"), + {"fields": ("twitter_oauth_token", "twitter_oauth_token_secret")}, + ), ( _("Permission settings"), {"classes": ("collapse",), "fields": ("is_staff", "is_superuser")}, diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html index ce565ec..c5ebea8 100644 --- a/src/newsreader/accounts/templates/accounts/views/integrations.html +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -44,13 +44,25 @@

    Twitter

    - + {% if twitter_request_url %} + + {% else %} + + {% endif %} - + {% if twitter_revoke_url %} + + {% else %} + + {% endif %}
    diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index 68f27a6..f94f5ec 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -20,6 +20,8 @@ from newsreader.accounts.views import ( RegistrationCompleteView, RegistrationView, SettingsView, + TwitterAuthRedirectView, + TwitterTemplateView, ) @@ -84,6 +86,16 @@ urlpatterns = [ login_required(RedditRevokeRedirectView.as_view()), name="reddit-revoke", ), + path( + "settings/integrations/twitter/request/", + login_required(TwitterAuthRedirectView.as_view()), + name="twitter-request", + ), + path( + "settings/integrations/twitter/callback/", + login_required(TwitterTemplateView.as_view()), + name="twitter-template", + ), path( "settings/integrations", login_required(IntegrationsView.as_view()), diff --git a/src/newsreader/accounts/views/__init__.py b/src/newsreader/accounts/views/__init__.py index 9325728..7c0b946 100644 --- a/src/newsreader/accounts/views/__init__.py +++ b/src/newsreader/accounts/views/__init__.py @@ -4,6 +4,8 @@ from newsreader.accounts.views.integrations import ( RedditRevokeRedirectView, RedditTemplateView, RedditTokenRedirectView, + TwitterAuthRedirectView, + TwitterTemplateView, ) from newsreader.accounts.views.password import ( PasswordChangeView, diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 3b30ba3..a8ec089 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -37,6 +37,7 @@ class IntegrationsView(TemplateView): return { **super().get_context_data(**kwargs), **self.get_reddit_context(**kwargs), + **self.get_twitter_context(**kwargs), } def get_reddit_context(self, **kwargs): @@ -66,6 +67,19 @@ class IntegrationsView(TemplateView): ), } + def get_twitter_context(self, **kwargs): + twitter_revoke_url = None + user = self.request.user + + # TODO add revoke url + if user.twitter_oauth_token and user.twitter_oauth_token_secret: + twitter_revoke_url = "https://foo/bar" + + return { + "twitter_request_url": reverse_lazy("accounts:twitter-request"), + "twitter_revoke_url": twitter_revoke_url, + } + class RedditTemplateView(TemplateView): template_name = "accounts/views/reddit.html" @@ -162,8 +176,6 @@ class RedditRevokeRedirectView(RedirectView): return response -# TODO hookup url to urlconf -# TODO hookup url to integrations button class TwitterAuthRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") @@ -182,8 +194,7 @@ class TwitterAuthRedirectView(RedirectView): messages.error(request, _("Unable to retrieve initial Twitter token")) return super().get(request, *args, **kwargs) - params = parse_qs(response.content) - + params = parse_qs(response.text) request_oauth_token = params.get("oauth_token")[0] request_oauth_secret = params.get("oauth_token_secret")[0] @@ -194,20 +205,20 @@ class TwitterAuthRedirectView(RedirectView): } ) - request_params = urlencode({"oauth_token": request_oauth_secret}) + request_params = urlencode({"oauth_token": request_oauth_token}) return redirect(f"{TWITTER_AUTH_URL}/?{request_params}") -# TODO hookup url +# TODO remove cached tokens class TwitterTemplateView(TemplateView): template_name = "accounts/views/twitter.html" def get(self, request, *args, **kwargs): context = self.get_context_data(**kwargs) - denied = request.GET.get("denied", True) + denied = request.GET.get("denied", False) oauth_token = request.GET.get("oauth_token") - request.GET.get("oauth_verifier") + oauth_verifier = request.GET.get("oauth_verifier") if denied: return self.render_to_response( @@ -231,7 +242,7 @@ class TwitterTemplateView(TemplateView): cached_secret = cache.get(f"twitter-{request.user.email}-secret") - if not cached_token or not cached_token_secret: + if not cached_token or not cached_secret: return self.render_to_response( { **context, @@ -245,7 +256,7 @@ class TwitterTemplateView(TemplateView): client_secret=settings.TWITTER_CONSUMER_SECRET, resource_owner_key=cached_token, resource_owner_secret=cached_secret, - verifier=oauth_token_secret, + verifier=oauth_verifier, ) try: @@ -261,7 +272,7 @@ class TwitterTemplateView(TemplateView): } ) - params = parse_qs(response.content) + params = parse_qs(response.text) oauth_token = params.get("oauth_token")[0] oauth_secret = params.get("oauth_token_secret")[0] From 885a0a52b726ba22c97bad33371c6a214176b93d Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Thu, 24 Sep 2020 23:01:00 +0200 Subject: [PATCH 48/69] Add Twitter revoke view --- src/newsreader/accounts/models.py | 4 ++ .../accounts/views/integrations.html | 2 +- src/newsreader/accounts/urls.py | 6 +++ src/newsreader/accounts/views/__init__.py | 1 + src/newsreader/accounts/views/integrations.py | 41 +++++++++++++++++-- src/newsreader/news/collection/twitter.py | 1 + 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/newsreader/accounts/models.py b/src/newsreader/accounts/models.py index a54c375..baf0b39 100644 --- a/src/newsreader/accounts/models.py +++ b/src/newsreader/accounts/models.py @@ -84,3 +84,7 @@ class User(AbstractUser): def delete(self, *args, **kwargs): self.task.delete() return super().delete(*args, **kwargs) + + @property + def has_twitter_auth(self): + return self.twitter_oauth_token and self.twitter_oauth_token_secret diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html index c5ebea8..8bb0b2e 100644 --- a/src/newsreader/accounts/templates/accounts/views/integrations.html +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -55,7 +55,7 @@ {% endif %} {% if twitter_revoke_url %} - {% else %} diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index f94f5ec..9f0f2e3 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -21,6 +21,7 @@ from newsreader.accounts.views import ( RegistrationView, SettingsView, TwitterAuthRedirectView, + TwitterRevokeRedirectView, TwitterTemplateView, ) @@ -96,6 +97,11 @@ urlpatterns = [ login_required(TwitterTemplateView.as_view()), name="twitter-template", ), + path( + "settings/integrations/twitter/revoke/", + login_required(TwitterRevokeRedirectView.as_view()), + name="twitter-revoke", + ), path( "settings/integrations", login_required(IntegrationsView.as_view()), diff --git a/src/newsreader/accounts/views/__init__.py b/src/newsreader/accounts/views/__init__.py index 7c0b946..81dd1fc 100644 --- a/src/newsreader/accounts/views/__init__.py +++ b/src/newsreader/accounts/views/__init__.py @@ -5,6 +5,7 @@ from newsreader.accounts.views.integrations import ( RedditTemplateView, RedditTokenRedirectView, TwitterAuthRedirectView, + TwitterRevokeRedirectView, TwitterTemplateView, ) from newsreader.accounts.views.password import ( diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index a8ec089..9a7c0be 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -23,6 +23,7 @@ from newsreader.news.collection.twitter import ( TWITTER_ACCESS_TOKEN_URL, TWITTER_AUTH_URL, TWITTER_REQUEST_TOKEN_URL, + TWITTER_REVOKE_URL, ) from newsreader.news.collection.utils import post @@ -69,11 +70,9 @@ class IntegrationsView(TemplateView): def get_twitter_context(self, **kwargs): twitter_revoke_url = None - user = self.request.user - # TODO add revoke url - if user.twitter_oauth_token and user.twitter_oauth_token_secret: - twitter_revoke_url = "https://foo/bar" + if self.request.user.has_twitter_auth: + twitter_revoke_url = reverse_lazy("accounts:twitter-revoke") return { "twitter_request_url": reverse_lazy("accounts:twitter-request"), @@ -176,6 +175,39 @@ class RedditRevokeRedirectView(RedirectView): return response +# TODO write tests +class TwitterRevokeRedirectView(RedirectView): + url = reverse_lazy("accounts:integrations") + + def get(self, request, *args, **kwargs): + if not request.user.has_twitter_auth: + messages.error(request, _("No twitter credentials found")) + return super().get(request, *args, **kwargs) + + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=request.user.twitter_oauth_token, + resource_owner_secret=request.user.twitter_oauth_token_secret, + ) + + try: + post(TWITTER_REVOKE_URL, auth=oauth) + except StreamException: + logger.exception("Failed revoking Twitter account") + + messages.error(request, _("Unable revoke Twitter account")) + return super().get(request, *args, **kwargs) + + request.user.twitter_oauth_token = None + request.user.twitter_oauth_token_secret = None + request.user.save() + + messages.success(request, _("Twitter account revoked")) + return super().get(request, *args, **kwargs) + + +# TODO write tests class TwitterAuthRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") @@ -210,6 +242,7 @@ class TwitterAuthRedirectView(RedirectView): # TODO remove cached tokens +# TODO write tests class TwitterTemplateView(TemplateView): template_name = "accounts/views/twitter.html" diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 7f075d7..bfd4cbf 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -34,6 +34,7 @@ TWITTER_API_URL = "https://api.twitter.com/1.1" TWITTER_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" TWITTER_AUTH_URL = "https://api.twitter.com/oauth/authorize" TWITTER_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" +TWITTER_REVOKE_URL = f"{TWITTER_API_URL}/oauth/invalidate_token" class TwitterBuilder(PostBuilder): From ef7997d3bc0cbb61ba0cb2dc72b9511a8caddbbc Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 25 Sep 2020 21:00:33 +0200 Subject: [PATCH 49/69] Add TwitterRevokeView tests --- .../accounts/tests/test_integrations.py | 53 +++++++++++++++++++ src/newsreader/accounts/views/integrations.py | 1 - 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py index 098a126..4bac037 100644 --- a/src/newsreader/accounts/tests/test_integrations.py +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -273,3 +273,56 @@ class RedditRevokeRedirectViewTestCase(TestCase): self.assertEquals(self.user.reddit_access_token, "jadajadajada") self.assertEquals(self.user.reddit_refresh_token, "jadajadajada") + + +class TwitterRevokeRedirectView(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + patch.stopall() + + def test_simple(self): + self.user.twitter_oauth_token = "jadajadajada" + self.user.twitter_oauth_token_secret = "jadajadajada" + self.user.save() + + response = self.client.get(reverse("accounts:twitter-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + def test_no_authorized_account(self): + self.user.twitter_oauth_token = None + self.user.twitter_oauth_token_secret = None + self.user.save() + + response = self.client.get(reverse("accounts:twitter-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.mocked_post.assert_not_called() + + def test_stream_exception(self): + self.user.twitter_oauth_token = "jadajadajada" + self.user.twitter_oauth_token_secret = "jadajadajada" + self.user.save() + + self.mocked_post.side_effect = StreamException + + response = self.client.get(reverse("accounts:twitter-revoke")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.twitter_oauth_token, "jadajadajada") + self.assertEquals(self.user.twitter_oauth_token_secret, "jadajadajada") diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 9a7c0be..2ff8c00 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -175,7 +175,6 @@ class RedditRevokeRedirectView(RedirectView): return response -# TODO write tests class TwitterRevokeRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") From c5ab5dc2bd381f40715580fad521f17b39c66c44 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 25 Sep 2020 21:23:27 +0200 Subject: [PATCH 50/69] Add TwitterAuth view tests --- .../accounts/views/integrations.html | 4 +- .../accounts/tests/test_integrations.py | 60 ++++++++++++++++++- src/newsreader/accounts/urls.py | 4 +- src/newsreader/accounts/views/integrations.py | 14 +++-- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/newsreader/accounts/templates/accounts/views/integrations.html b/src/newsreader/accounts/templates/accounts/views/integrations.html index 8bb0b2e..4429f02 100644 --- a/src/newsreader/accounts/templates/accounts/views/integrations.html +++ b/src/newsreader/accounts/templates/accounts/views/integrations.html @@ -44,8 +44,8 @@

    Twitter

    - {% if twitter_request_url %} - {% else %} diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py index 4bac037..5c65caf 100644 --- a/src/newsreader/accounts/tests/test_integrations.py +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import Mock, patch from urllib.parse import urlencode from uuid import uuid4 @@ -13,6 +13,7 @@ from newsreader.news.collection.exceptions import ( StreamException, StreamTooManyException, ) +from newsreader.news.collection.twitter import TWITTER_AUTH_URL class IntegrationsViewTestCase(TestCase): @@ -326,3 +327,60 @@ class TwitterRevokeRedirectView(TestCase): self.assertEquals(self.user.twitter_oauth_token, "jadajadajada") self.assertEquals(self.user.twitter_oauth_token_secret, "jadajadajada") + + +class TwitterAuthRedirectViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + cache.clear() + + def test_simple(self): + self.mocked_post.return_value = Mock( + text="oauth_token=foo&oauth_token_secret=bar" + ) + + response = self.client.get(reverse("accounts:twitter-auth")) + + self.assertRedirects( + response, + f"{TWITTER_AUTH_URL}/?oauth_token=foo", + fetch_redirect_response=False, + ) + + cached_token = cache.get(f"twitter-{self.user.email}-token") + cached_secret = cache.get(f"twitter-{self.user.email}-secret") + + self.assertEquals(cached_token, "foo") + self.assertEquals(cached_secret, "bar") + + def test_stream_exception(self): + self.mocked_post.side_effect = StreamException + + response = self.client.get(reverse("accounts:twitter-auth")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + cached_token = cache.get(f"twitter-{self.user.email}-token") + cached_secret = cache.get(f"twitter-{self.user.email}-secret") + + self.assertIsNone(cached_token) + self.assertIsNone(cached_secret) + + def test_unexpected_contents(self): + self.mocked_post.return_value = Mock(text="foo=bar&oauth_token_secret=bar") + + response = self.client.get(reverse("accounts:twitter-auth")) + + self.assertRedirects(response, reverse("accounts:integrations")) + + cached_token = cache.get(f"twitter-{self.user.email}-token") + cached_secret = cache.get(f"twitter-{self.user.email}-secret") + + self.assertIsNone(cached_token) + self.assertIsNone(cached_secret) diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index 9f0f2e3..841e40a 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -88,9 +88,9 @@ urlpatterns = [ name="reddit-revoke", ), path( - "settings/integrations/twitter/request/", + "settings/integrations/twitter/auth/", login_required(TwitterAuthRedirectView.as_view()), - name="twitter-request", + name="twitter-auth", ), path( "settings/integrations/twitter/callback/", diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 2ff8c00..a146bda 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -75,7 +75,7 @@ class IntegrationsView(TemplateView): twitter_revoke_url = reverse_lazy("accounts:twitter-revoke") return { - "twitter_request_url": reverse_lazy("accounts:twitter-request"), + "twitter_auth_url": reverse_lazy("accounts:twitter-auth"), "twitter_revoke_url": twitter_revoke_url, } @@ -206,7 +206,6 @@ class TwitterRevokeRedirectView(RedirectView): return super().get(request, *args, **kwargs) -# TODO write tests class TwitterAuthRedirectView(RedirectView): url = reverse_lazy("accounts:integrations") @@ -226,8 +225,15 @@ class TwitterAuthRedirectView(RedirectView): return super().get(request, *args, **kwargs) params = parse_qs(response.text) - request_oauth_token = params.get("oauth_token")[0] - request_oauth_secret = params.get("oauth_token_secret")[0] + + try: + request_oauth_token = params["oauth_token"][0] + request_oauth_secret = params["oauth_token_secret"][0] + except KeyError: + logger.exception("No credentials found in response") + + messages.error(request, _("Unable to retrieve initial Twitter token")) + return super().get(request, *args, **kwargs) cache.set_many( { From 465e5933681975838a00c6c027967664d038d18e Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 25 Sep 2020 22:34:56 +0200 Subject: [PATCH 51/69] Add TwitterTemplate view tests --- .../accounts/tests/test_integrations.py | 151 ++++++++++++++++++ src/newsreader/accounts/views/integrations.py | 25 ++- 2 files changed, 172 insertions(+), 4 deletions(-) diff --git a/src/newsreader/accounts/tests/test_integrations.py b/src/newsreader/accounts/tests/test_integrations.py index 5c65caf..cdc9546 100644 --- a/src/newsreader/accounts/tests/test_integrations.py +++ b/src/newsreader/accounts/tests/test_integrations.py @@ -5,6 +5,7 @@ from uuid import uuid4 from django.core.cache import cache from django.test import TestCase from django.urls import reverse +from django.utils.translation import gettext as _ from bs4 import BeautifulSoup @@ -384,3 +385,153 @@ class TwitterAuthRedirectViewTestCase(TestCase): self.assertIsNone(cached_token) self.assertIsNone(cached_secret) + + +class TwitterTemplateViewTestCase(TestCase): + def setUp(self): + self.user = UserFactory(email="test@test.nl", password="test") + self.client.force_login(self.user) + + self.patch = patch("newsreader.accounts.views.integrations.post") + self.mocked_post = self.patch.start() + + def tearDown(self): + cache.clear() + + def test_simple(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + self.mocked_post.return_value = Mock( + text="oauth_token=realtoken&oauth_token_secret=realsecret" + ) + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("Twitter account is linked")) + + self.user.refresh_from_db() + + self.assertEquals(self.user.twitter_oauth_token, "realtoken") + self.assertEquals(self.user.twitter_oauth_token_secret, "realsecret") + + self.assertIsNone(cache.get(f"twitter-{self.user.email}-token")) + self.assertIsNone(cache.get(f"twitter-{self.user.email}-secret")) + + def test_denied(self): + params = {"denied": "true", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("Twitter authorization failed")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.mocked_post.assert_not_called() + + def test_mismatched_token(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "boo", "oauth_verifier": "barfoo"} + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("OAuth tokens failed to match")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.mocked_post.assert_not_called() + + def test_missing_secret(self): + cache.set_many({f"twitter-{self.user.email}-token": "foo"}) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("No matching tokens found for this user")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.mocked_post.assert_not_called() + + def test_stream_exception(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + self.mocked_post.side_effect = StreamException + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("Failed requesting access token")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-token")) + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-secret")) + + def test_unexpected_contents(self): + cache.set_many( + { + f"twitter-{self.user.email}-token": "foo", + f"twitter-{self.user.email}-secret": "bar", + } + ) + + params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"} + + self.mocked_post.return_value = Mock( + text="foobar=boo&oauth_token_secret=realsecret" + ) + + response = self.client.get( + f"{reverse('accounts:twitter-template')}?{urlencode(params)}" + ) + + self.assertContains(response, _("No credentials found in Twitter response")) + + self.user.refresh_from_db() + + self.assertIsNone(self.user.twitter_oauth_token) + self.assertIsNone(self.user.twitter_oauth_token_secret) + + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-token")) + self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-secret")) diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index a146bda..1b25ab8 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -246,8 +246,6 @@ class TwitterAuthRedirectView(RedirectView): return redirect(f"{TWITTER_AUTH_URL}/?{request_params}") -# TODO remove cached tokens -# TODO write tests class TwitterTemplateView(TemplateView): template_name = "accounts/views/twitter.html" @@ -311,11 +309,30 @@ class TwitterTemplateView(TemplateView): ) params = parse_qs(response.text) - oauth_token = params.get("oauth_token")[0] - oauth_secret = params.get("oauth_token_secret")[0] + + try: + oauth_token = params["oauth_token"][0] + oauth_secret = params["oauth_token_secret"][0] + except KeyError: + logger.exception("No credentials in Twitter response") + + return self.render_to_response( + { + **context, + "error": _("No credentials found in Twitter response"), + "authorized": False, + } + ) request.user.twitter_oauth_token = oauth_token request.user.twitter_oauth_token_secret = oauth_secret request.user.save() + cache.delete_many( + [ + f"twitter-{request.user.email}-token", + f"twitter-{request.user.email}-secret", + ] + ) + return self.render_to_response({**context, "error": None, "authorized": True}) From 66c3cc2154553001464664183b65f31106461ad1 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Fri, 25 Sep 2020 22:37:30 +0200 Subject: [PATCH 52/69] Use translatable error messages --- src/newsreader/accounts/views/integrations.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/newsreader/accounts/views/integrations.py b/src/newsreader/accounts/views/integrations.py index 1b25ab8..62d71fc 100644 --- a/src/newsreader/accounts/views/integrations.py +++ b/src/newsreader/accounts/views/integrations.py @@ -102,7 +102,9 @@ class RedditTemplateView(TemplateView): return self.render_to_response( { **context, - "error": "The saved state for Reddit authorization did not match", + "error": _( + "The saved state for Reddit authorization did not match" + ), } ) @@ -120,7 +122,10 @@ class RedditTemplateView(TemplateView): return self.render_to_response({**context, "error": str(e)}) except KeyError: return self.render_to_response( - {**context, "error": "Access and refresh token not found in response"} + { + **context, + "error": _("Access and refresh token not found in response"), + } ) @@ -260,7 +265,7 @@ class TwitterTemplateView(TemplateView): return self.render_to_response( { **context, - "error": "Twitter authorization failed", + "error": _("Twitter authorization failed"), "authorized": False, } ) @@ -271,7 +276,7 @@ class TwitterTemplateView(TemplateView): return self.render_to_response( { **context, - "error": "OAuth tokens failed to match", + "error": _("OAuth tokens failed to match"), "authorized": False, } ) @@ -282,7 +287,7 @@ class TwitterTemplateView(TemplateView): return self.render_to_response( { **context, - "error": "No matching tokens found for this user", + "error": _("No matching tokens found for this user"), "authorized": False, } ) @@ -303,7 +308,7 @@ class TwitterTemplateView(TemplateView): return self.render_to_response( { **context, - "error": "Failed requesting access token", + "error": _("Failed requesting access token"), "authorized": False, } ) From 14fb17b33425b3ee81322310f1c7966fc9c8b118 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 17:08:06 +0200 Subject: [PATCH 53/69] Update scheduler & collector --- src/newsreader/news/collection/twitter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index bfd4cbf..f7aafad 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -156,7 +156,8 @@ class TwitterClient(PostClient): class TwitterCollector(PostCollector): - pass + builder = TwitterBuilder + client = TwitterClient # see https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits @@ -182,9 +183,7 @@ class TwitterTimeLineScheduler(Scheduler): return self.timelines[:max_amount] if max_amount else [] def get_current_ratelimit(self): - endpoint = ( - "application/rate_limit_status.json?resources=help,users,search,statuses" - ) + endpoint = "application/rate_limit_status.json?resources=statuses" # TODO add appropriate authentication (OAuth 1.0a) headers try: From c830a42cc6606b998efd5e0d46b365027e9def92 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 17:25:32 +0200 Subject: [PATCH 54/69] Add TwitterTimeline task upon Twitter timeline creation --- .../tests/views/test_twitter_views.py | 9 ++++++- .../news/collection/views/twitter.py | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/tests/views/test_twitter_views.py b/src/newsreader/news/collection/tests/views/test_twitter_views.py index 4f178cc..35897be 100644 --- a/src/newsreader/news/collection/tests/views/test_twitter_views.py +++ b/src/newsreader/news/collection/tests/views/test_twitter_views.py @@ -1,9 +1,10 @@ - from django.test import TestCase from django.urls import reverse import pytz +from django_celery_beat.models import PeriodicTask + from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.tests.factories import TwitterTimelineFactory @@ -38,6 +39,12 @@ class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): self.assertEquals(rule.category.pk, self.category.pk) self.assertEquals(rule.user.pk, self.user.pk) + self.assertTrue( + PeriodicTask.objects.get( + name=self.user.email, task="TwitterTimelineTask", enabled=True + ) + ) + class TwitterTimelineUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): def setUp(self): diff --git a/src/newsreader/news/collection/views/twitter.py b/src/newsreader/news/collection/views/twitter.py index c60dbda..563bddf 100644 --- a/src/newsreader/news/collection/views/twitter.py +++ b/src/newsreader/news/collection/views/twitter.py @@ -1,5 +1,9 @@ +import json + from django.views.generic.edit import CreateView, UpdateView +from django_celery_beat.models import IntervalSchedule, PeriodicTask + from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.forms import TwitterTimelineForm from newsreader.news.collection.views.base import ( @@ -14,6 +18,28 @@ class TwitterTimelineCreateView( form_class = TwitterTimelineForm template_name = "news/collection/views/twitter/timeline-create.html" + def form_valid(self, form): + response = super().form_valid(form) + + task_interval, _ = IntervalSchedule.objects.get_or_create( + every=1, period=IntervalSchedule.HOURS + ) + + task, _ = PeriodicTask.objects.get_or_create( + name=self.request.user.email, + task="TwitterTimelineTask", + enabled=True, + defaults={ + "args": json.dumps([self.request.user.pk]), + "interval": task_interval, + }, + ) + + self.request.user.task = task + self.request.user.save() + + return response + class TwitterTimelineUpdateView( CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView From 8c69a22b3264cd2cdc557860c05c00ea1e28299f Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 18:51:40 +0200 Subject: [PATCH 55/69] Update task creation --- .../migrations/0012_remove_user_task.py | 10 ++++++ src/newsreader/accounts/models.py | 34 +++---------------- src/newsreader/accounts/tests/tests.py | 26 +++++++------- src/newsreader/news/collection/reddit.py | 2 +- .../news/collection/tests/views/test_crud.py | 8 +++++ .../tests/views/test_twitter_views.py | 4 ++- src/newsreader/news/collection/views/base.py | 26 ++++++++++++++ src/newsreader/news/collection/views/feed.py | 12 +++++-- .../news/collection/views/twitter.py | 31 ++++------------- 9 files changed, 82 insertions(+), 71 deletions(-) create mode 100644 src/newsreader/accounts/migrations/0012_remove_user_task.py diff --git a/src/newsreader/accounts/migrations/0012_remove_user_task.py b/src/newsreader/accounts/migrations/0012_remove_user_task.py new file mode 100644 index 0000000..250d300 --- /dev/null +++ b/src/newsreader/accounts/migrations/0012_remove_user_task.py @@ -0,0 +1,10 @@ +# Generated by Django 3.0.7 on 2020-09-26 15:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("accounts", "0011_auto_20200913_2101")] + + operations = [migrations.RemoveField(model_name="user", name="task")] diff --git a/src/newsreader/accounts/models.py b/src/newsreader/accounts/models.py index baf0b39..6341385 100644 --- a/src/newsreader/accounts/models.py +++ b/src/newsreader/accounts/models.py @@ -1,11 +1,10 @@ -import json from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import UserManager as DjangoUserManager from django.db import models from django.utils.translation import gettext as _ -from django_celery_beat.models import IntervalSchedule, PeriodicTask +from django_celery_beat.models import PeriodicTask class UserManager(DjangoUserManager): @@ -41,15 +40,6 @@ class UserManager(DjangoUserManager): class User(AbstractUser): email = models.EmailField(_("email address"), unique=True) - task = models.OneToOneField( - PeriodicTask, - on_delete=models.CASCADE, - null=True, - blank=True, - editable=False, - verbose_name="collection task", - ) - reddit_refresh_token = models.CharField(max_length=255, blank=True, null=True) reddit_access_token = models.CharField(max_length=255, blank=True, null=True) @@ -63,26 +53,10 @@ class User(AbstractUser): USERNAME_FIELD = "email" REQUIRED_FIELDS = [] - def save(self, *args, **kwargs): - super().save(*args, **kwargs) - - if not self.task: - task_interval, _ = IntervalSchedule.objects.get_or_create( - every=1, period=IntervalSchedule.HOURS - ) - - self.task, _ = PeriodicTask.objects.get_or_create( - enabled=True, - interval=task_interval, - name=f"{self.email}-collection-task", - task="FeedTask", - args=json.dumps([self.pk]), - ) - - self.save() - def delete(self, *args, **kwargs): - self.task.delete() + tasks = PeriodicTask.objects.filter(name__contains=self.email) + tasks.delete() + return super().delete(*args, **kwargs) @property diff --git a/src/newsreader/accounts/tests/tests.py b/src/newsreader/accounts/tests/tests.py index e28dbd3..9f6a20f 100644 --- a/src/newsreader/accounts/tests/tests.py +++ b/src/newsreader/accounts/tests/tests.py @@ -1,22 +1,24 @@ from django.test import TestCase -from django_celery_beat.models import PeriodicTask +from django_celery_beat.models import IntervalSchedule, PeriodicTask -from newsreader.accounts.models import User +from newsreader.accounts.tests.factories import UserFactory class UserTestCase(TestCase): - def test_task_is_created(self): - user = User.objects.create(email="durp@burp.nl", task=None) - task = PeriodicTask.objects.get(name=f"{user.email}-collection-task") - - user.refresh_from_db() - - self.assertEquals(task, user.task) - self.assertEquals(PeriodicTask.objects.count(), 1) - def test_task_is_deleted(self): - user = User.objects.create(email="durp@burp.nl", task=None) + user = UserFactory(email="durp@burp.nl") + + interval = IntervalSchedule.objects.create( + every=1, period=IntervalSchedule.HOURS + ) + PeriodicTask.objects.create( + name=f"{user.email}-feed", task="FeedTask", interval=interval + ) + PeriodicTask.objects.create( + name=f"{user.email}-timeline", task="TwitterTimelineTask", interval=interval + ) + user.delete() self.assertEquals(PeriodicTask.objects.count(), 0) diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 1cb9e29..22af39f 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -290,7 +290,7 @@ class RedditClient(PostClient): break except StreamException as e: logger.exception( - "Stream failed reading content from {stream.rule.url}" + f"Stream failed reading content from {stream.rule.url}" ) self.set_rule_error(stream.rule, e) diff --git a/src/newsreader/news/collection/tests/views/test_crud.py b/src/newsreader/news/collection/tests/views/test_crud.py index e10f997..7da241d 100644 --- a/src/newsreader/news/collection/tests/views/test_crud.py +++ b/src/newsreader/news/collection/tests/views/test_crud.py @@ -3,6 +3,8 @@ from django.urls import reverse import pytz +from django_celery_beat.models import PeriodicTask + from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.tests.factories import FeedFactory @@ -37,6 +39,12 @@ class FeedCreateViewTestCase(CollectionRuleViewTestCase, TestCase): self.assertEquals(rule.category.pk, self.category.pk) self.assertEquals(rule.user.pk, self.user.pk) + self.assertTrue( + PeriodicTask.objects.get( + name=f"{self.user.email}-feed", task="FeedTask", enabled=True + ) + ) + class FeedUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): def setUp(self): diff --git a/src/newsreader/news/collection/tests/views/test_twitter_views.py b/src/newsreader/news/collection/tests/views/test_twitter_views.py index 35897be..129ed79 100644 --- a/src/newsreader/news/collection/tests/views/test_twitter_views.py +++ b/src/newsreader/news/collection/tests/views/test_twitter_views.py @@ -41,7 +41,9 @@ class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): self.assertTrue( PeriodicTask.objects.get( - name=self.user.email, task="TwitterTimelineTask", enabled=True + name=f"{self.user.email}-timeline", + task="TwitterTimelineTask", + enabled=True, ) ) diff --git a/src/newsreader/news/collection/views/base.py b/src/newsreader/news/collection/views/base.py index 982a4d3..8a9207d 100644 --- a/src/newsreader/news/collection/views/base.py +++ b/src/newsreader/news/collection/views/base.py @@ -1,7 +1,11 @@ +import json + from django.urls import reverse_lazy import pytz +from django_celery_beat.models import IntervalSchedule, PeriodicTask + from newsreader.news.collection.models import CollectionRule from newsreader.news.core.models import Category @@ -32,3 +36,25 @@ class CollectionRuleDetailMixin: kwargs = super().get_form_kwargs() kwargs["user"] = self.request.user return kwargs + + +class TaskCreationMixin: + def form_valid(self, form): + response = super().form_valid(form) + + interval, period = self.task_interval + task_interval, _ = IntervalSchedule.objects.get_or_create( + every=interval, period=period + ) + + PeriodicTask.objects.get_or_create( + name=f"{self.request.user.email}-{self.task_name}", + task=self.task_type, + enabled=True, + defaults={ + "args": json.dumps([self.request.user.pk]), + "interval": task_interval, + }, + ) + + return response diff --git a/src/newsreader/news/collection/views/feed.py b/src/newsreader/news/collection/views/feed.py index 872d716..b7803d2 100644 --- a/src/newsreader/news/collection/views/feed.py +++ b/src/newsreader/news/collection/views/feed.py @@ -3,6 +3,8 @@ from django.urls import reverse from django.utils.translation import gettext as _ from django.views.generic.edit import CreateView, FormView, UpdateView +from django_celery_beat.models import IntervalSchedule + from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.forms import ( CollectionRuleBulkForm, @@ -13,12 +15,14 @@ from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.views.base import ( CollectionRuleDetailMixin, CollectionRuleViewMixin, + TaskCreationMixin, ) from newsreader.utils.opml import parse_opml class FeedUpdateView(CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView): template_name = "news/collection/views/feed-update.html" + context_object_name = "feed" form_class = FeedForm def get_queryset(self): @@ -26,10 +30,14 @@ class FeedUpdateView(CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateV return queryset.filter(type=RuleTypeChoices.feed) -class FeedCreateView(CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView): +class FeedCreateView( + CollectionRuleViewMixin, CollectionRuleDetailMixin, TaskCreationMixin, CreateView +): template_name = "news/collection/views/feed-create.html" + task_interval = (1, IntervalSchedule.HOURS) + task_name = "feed" + task_type = "FeedTask" form_class = FeedForm - context_object_name = "feed" class OPMLImportView(FormView): diff --git a/src/newsreader/news/collection/views/twitter.py b/src/newsreader/news/collection/views/twitter.py index 563bddf..1755b49 100644 --- a/src/newsreader/news/collection/views/twitter.py +++ b/src/newsreader/news/collection/views/twitter.py @@ -1,44 +1,25 @@ -import json from django.views.generic.edit import CreateView, UpdateView -from django_celery_beat.models import IntervalSchedule, PeriodicTask +from django_celery_beat.models import IntervalSchedule from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.forms import TwitterTimelineForm from newsreader.news.collection.views.base import ( CollectionRuleDetailMixin, CollectionRuleViewMixin, + TaskCreationMixin, ) class TwitterTimelineCreateView( - CollectionRuleViewMixin, CollectionRuleDetailMixin, CreateView + CollectionRuleViewMixin, CollectionRuleDetailMixin, TaskCreationMixin, CreateView ): form_class = TwitterTimelineForm template_name = "news/collection/views/twitter/timeline-create.html" - - def form_valid(self, form): - response = super().form_valid(form) - - task_interval, _ = IntervalSchedule.objects.get_or_create( - every=1, period=IntervalSchedule.HOURS - ) - - task, _ = PeriodicTask.objects.get_or_create( - name=self.request.user.email, - task="TwitterTimelineTask", - enabled=True, - defaults={ - "args": json.dumps([self.request.user.pk]), - "interval": task_interval, - }, - ) - - self.request.user.task = task - self.request.user.save() - - return response + task_interval = (10, IntervalSchedule.MINUTES) + task_name = "timeline" + task_type = "TwitterTimelineTask" class TwitterTimelineUpdateView( From e21c8759289aba702293b6e3fc9515401380fdcd Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 19:34:10 +0200 Subject: [PATCH 56/69] Add twitter scheduler tests --- .../tests/twitter/test_scheduler.py | 63 +++++++++++++++++++ src/newsreader/news/collection/twitter.py | 42 ++++++++----- src/newsreader/news/collection/utils.py | 4 +- 3 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 src/newsreader/news/collection/tests/twitter/test_scheduler.py diff --git a/src/newsreader/news/collection/tests/twitter/test_scheduler.py b/src/newsreader/news/collection/tests/twitter/test_scheduler.py new file mode 100644 index 0000000..a3c2db8 --- /dev/null +++ b/src/newsreader/news/collection/tests/twitter/test_scheduler.py @@ -0,0 +1,63 @@ +from json import JSONDecodeError +from unittest.mock import patch + +from django.test import TestCase + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.news.collection.exceptions import StreamException +from newsreader.news.collection.twitter import TwitterTimeLineScheduler + + +class TwitterTimeLineSchedulerTestCase(TestCase): + def setUp(self): + patched_fetch = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = patched_fetch.start() + + def test_simple(self): + user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar") + + self.mocked_fetch.return_value.json.return_value = { + "rate_limit_context": {"application": "dummykey"}, + "resources": { + "statuses": { + "/statuses/user_timeline": { + "limit": 1500, + "remaining": 1500, + "reset": 1601141386, + } + } + }, + } + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), 1500) + + def test_stream_exception(self): + user = UserFactory(twitter_oauth_token=None, twitter_oauth_token_secret=None) + + self.mocked_fetch.side_effect = StreamException + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), None) + + def test_json_decode_error(self): + user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar") + + self.mocked_fetch.return_value.json.side_effect = JSONDecodeError( + "foo", "bar", 10 + ) + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), None) + + def test_unexpected_contents(self): + user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar") + + self.mocked_fetch.return_value.json.return_value = {"foo": "bar"} + + scheduler = TwitterTimeLineScheduler(user) + + self.assertEquals(scheduler.get_current_ratelimit(), None) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index f7aafad..218c560 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -3,11 +3,13 @@ import logging from datetime import datetime from json import JSONDecodeError +from django.conf import settings from django.utils.html import format_html, urlize import pytz from ftfy import fix_text +from requests_oauthlib import OAuth1 as OAuth from newsreader.news.collection.base import ( PostBuilder, @@ -166,28 +168,36 @@ class TwitterTimeLineScheduler(Scheduler): self.user = user if not timelines: - self.timelines = user.rules.enabled( - type=RuleTypeChoices.twitter_timeline - ).order_by("last_run")[:200] + self.timelines = ( + user.rules.enabled() + .filter(type=RuleTypeChoices.twitter_timeline) + .order_by("last_run")[:200] + ) else: self.timelines = timelines def get_scheduled_rules(self): - if ( - not self.user.twitter_oauth_token - or not self.user.twitter_oauth_token_secret - ): - return [] - max_amount = self.get_current_ratelimit() return self.timelines[:max_amount] if max_amount else [] def get_current_ratelimit(self): endpoint = "application/rate_limit_status.json?resources=statuses" - # TODO add appropriate authentication (OAuth 1.0a) headers + if ( + not self.user.twitter_oauth_token + or not self.user.twitter_oauth_token_secret + ): + return + + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=self.user.twitter_oauth_token, + resource_owner_secret=self.user.twitter_oauth_token_secret, + ) + try: - response = fetch(f"{TWITTER_API_URL}/{endpoint}") + response = fetch(f"{TWITTER_API_URL}/{endpoint}", auth=oauth) except StreamException: logger.exception(f"Unable to retrieve current ratelimit for {self.user.pk}") return @@ -198,9 +208,7 @@ class TwitterTimeLineScheduler(Scheduler): logger.exception(f"Unable to parse ratelimit request for {self.user.pk}") return - if not "resources" in payload or not "statuses" in payload["resources"]: - return [] - - statuses = payload["resources"]["statuses"] - - return statuses.get("/statuses/user_timeline", 0) + try: + return payload["resources"]["statuses"]["/statuses/user_timeline"]["limit"] + except KeyError: + return diff --git a/src/newsreader/news/collection/utils.py b/src/newsreader/news/collection/utils.py index 4cfc0e7..0eb1dc0 100644 --- a/src/newsreader/news/collection/utils.py +++ b/src/newsreader/news/collection/utils.py @@ -25,12 +25,12 @@ def build_publication_date(dt, tz): return published_parsed.astimezone(pytz.utc) -def fetch(url, headers={}): +def fetch(url, auth=None, headers={}): headers = {**DEFAULT_HEADERS, **headers} with ResponseHandler() as response_handler: try: - response = requests.get(url, headers=headers) + response = requests.get(url, auth=auth, headers=headers) response_handler.handle_response(response) except RequestException as exception: response_handler.map_exception(exception) From 3db3a05ca0b12adc00281c7a2ba331ef19545765 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 20:32:13 +0200 Subject: [PATCH 57/69] Add TwitterStream & tests --- src/newsreader/news/collection/reddit.py | 2 - .../collection/tests/twitter/stream/mocks.py | 225 ++++++++++++++++++ .../collection/tests/twitter/stream/tests.py | 107 +++++++++ src/newsreader/news/collection/twitter.py | 22 +- 4 files changed, 353 insertions(+), 3 deletions(-) diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index 22af39f..a306f17 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -219,12 +219,10 @@ class RedditBuilder(PostBuilder): class RedditStream(PostStream): rule_type = RuleTypeChoices.subreddit headers = {} - user = None def __init__(self, rule): super().__init__(rule) - self.user = self.rule.user self.headers = { f"Authorization": f"bearer {self.rule.user.reddit_access_token}" } diff --git a/src/newsreader/news/collection/tests/twitter/stream/mocks.py b/src/newsreader/news/collection/tests/twitter/stream/mocks.py index e69de29..1b7c6a2 100644 --- a/src/newsreader/news/collection/tests/twitter/stream/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/stream/mocks.py @@ -0,0 +1,225 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] diff --git a/src/newsreader/news/collection/tests/twitter/stream/tests.py b/src/newsreader/news/collection/tests/twitter/stream/tests.py index e69de29..4edb639 100644 --- a/src/newsreader/news/collection/tests/twitter/stream/tests.py +++ b/src/newsreader/news/collection/tests/twitter/stream/tests.py @@ -0,0 +1,107 @@ +from json import JSONDecodeError +from unittest.mock import patch + +from django.test import TestCase + +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamForbiddenException, + StreamNotFoundException, + StreamParseException, + StreamTimeOutException, + StreamTooManyException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.stream.mocks import simple_mock +from newsreader.news.collection.twitter import TwitterStream + + +class TwitterStreamTestCase(TestCase): + def setUp(self): + self.patched_fetch = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = self.patched_fetch.start() + + def tearDown(self): + patch.stopall() + + def test_simple_stream(self): + self.mocked_fetch.return_value.json.return_value = simple_mock + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + data, stream = stream.read() + + self.assertEquals(data, simple_mock) + self.assertEquals(stream, stream) + + self.mocked_fetch.assert_called() + + def test_stream_raises_exception(self): + self.mocked_fetch.side_effect = StreamException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_denied_exception(self): + self.mocked_fetch.side_effect = StreamDeniedException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamDeniedException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_not_found_exception(self): + self.mocked_fetch.side_effect = StreamNotFoundException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamNotFoundException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_time_out_exception(self): + self.mocked_fetch.side_effect = StreamTimeOutException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamTimeOutException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_forbidden_exception(self): + self.mocked_fetch.side_effect = StreamForbiddenException + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamForbiddenException): + stream.read() + + self.mocked_fetch.assert_called() + + def test_stream_raises_parse_exception(self): + self.mocked_fetch.return_value.json.side_effect = JSONDecodeError( + "No json found", "{}", 5 + ) + + timeline = TwitterTimelineFactory() + stream = TwitterStream(timeline) + + with self.assertRaises(StreamParseException): + stream.read() + + self.mocked_fetch.assert_called() diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 218c560..e2be78f 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -150,7 +150,27 @@ class TwitterBuilder(PostBuilder): class TwitterStream(PostStream): - pass + rule_type = RuleTypeChoices.subreddit + + def read(self): + oauth = OAuth( + settings.TWITTER_CONSUMER_ID, + client_secret=settings.TWITTER_CONSUMER_SECRET, + resource_owner_key=self.rule.user.twitter_oauth_token, + resource_owner_secret=self.rule.user.twitter_oauth_token_secret, + ) + + response = fetch(self.rule.url, auth=oauth) + + return self.parse(response), self + + def parse(self, response): + try: + return response.json() + except JSONDecodeError as e: + raise StreamParseException( + response=response, message="Failed parsing json" + ) from e class TwitterClient(PostClient): From 02bb95fde86ff9a9ec38da7f91f28b02f6cb7eda Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 20:44:00 +0200 Subject: [PATCH 58/69] Add twitter client --- src/newsreader/news/collection/twitter.py | 54 ++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index e2be78f..b503636 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -1,9 +1,11 @@ import logging +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from json import JSONDecodeError from django.conf import settings +from django.utils import timezone from django.utils.html import format_html, urlize import pytz @@ -22,7 +24,9 @@ from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeC from newsreader.news.collection.exceptions import ( StreamDeniedException, StreamException, + StreamNotFoundException, StreamParseException, + StreamTimeOutException, StreamTooManyException, ) from newsreader.news.collection.utils import fetch, truncate_text @@ -174,7 +178,55 @@ class TwitterStream(PostStream): class TwitterClient(PostClient): - pass + stream = TwitterStream + + def __enter__(self): + streams = [self.stream(timeline) for timeline in self.rules] + + with ThreadPoolExecutor(max_workers=10) as executor: + futures = {executor.submit(stream.read): stream for stream in streams} + + for future in as_completed(futures): + stream = futures[future] + + try: + payload = future.result() + + stream.rule.error = None + stream.rule.succeeded = True + + yield payload + except StreamTooManyException as e: + logger.exception("Ratelimit hit, aborting twitter calls") + + self.set_rule_error(stream.rule, e) + + break + except StreamDeniedException as e: + logger.warning(f"Access token expired for user {stream.user.pk}") + + stream.rule.user.twitter_oauth_token = None + stream.rule.user.twitter_oauth_token_secret = None + stream.rule.user.save() + + self.set_rule_error(stream.rule, e) + + break + except (StreamNotFoundException, StreamTimeOutException) as e: + logger.warning(f"Request failed for {stream.rule.screen_name}") + + self.set_rule_error(stream.rule, e) + + continue + except StreamException as e: + logger.exception(f"Request failed for {stream.rule.screen_name}") + + self.set_rule_error(stream.rule, e) + + continue + finally: + stream.rule.last_run = timezone.now() + stream.rule.save() class TwitterCollector(PostCollector): From 2a54556e49e4d1b018dd26c113d33fb5569de94c Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 21:18:38 +0200 Subject: [PATCH 59/69] Fix reddit tests --- src/newsreader/news/collection/reddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/reddit.py b/src/newsreader/news/collection/reddit.py index a306f17..daeb85f 100644 --- a/src/newsreader/news/collection/reddit.py +++ b/src/newsreader/news/collection/reddit.py @@ -268,7 +268,7 @@ class RedditClient(PostClient): yield response_data except StreamDeniedException as e: logger.warning( - f"Access token expired for user {stream.user.pk}" + f"Access token expired for user {stream.rule.user.pk}" ) stream.rule.user.reddit_access_token = None From 7745d29b254d0fe2174ea523d6de512d773917dd Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 21:34:03 +0200 Subject: [PATCH 60/69] Add TwitterClient tests --- .../collection/tests/twitter/client/mocks.py | 225 ++++++++++++++++++ .../collection/tests/twitter/client/tests.py | 162 +++++++++++++ src/newsreader/news/collection/twitter.py | 4 +- 3 files changed, 390 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/tests/twitter/client/mocks.py b/src/newsreader/news/collection/tests/twitter/client/mocks.py index e69de29..1b7c6a2 100644 --- a/src/newsreader/news/collection/tests/twitter/client/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/client/mocks.py @@ -0,0 +1,225 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] diff --git a/src/newsreader/news/collection/tests/twitter/client/tests.py b/src/newsreader/news/collection/tests/twitter/client/tests.py index e69de29..387ffef 100644 --- a/src/newsreader/news/collection/tests/twitter/client/tests.py +++ b/src/newsreader/news/collection/tests/twitter/client/tests.py @@ -0,0 +1,162 @@ +from unittest.mock import Mock, patch +from uuid import uuid4 + +from django.test import TestCase +from django.utils.lorem_ipsum import words + +from newsreader.accounts.tests.factories import UserFactory +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamException, + StreamNotFoundException, + StreamParseException, + StreamTimeOutException, + StreamTooManyException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.twitter import TwitterClient + +from .mocks import simple_mock + + +class TwitterClientTestCase(TestCase): + def setUp(self): + patched_read = patch("newsreader.news.collection.twitter.TwitterStream.read") + self.mocked_read = patched_read.start() + + def tearDown(self): + patch.stopall() + + def test_simple(self): + timeline = TwitterTimelineFactory() + mock_stream = Mock(rule=timeline) + + self.mocked_read.return_value = (simple_mock, mock_stream) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertEquals(data, simple_mock) + self.assertEquals(stream, mock_stream) + + self.mocked_read.assert_called() + + def test_client_catches_stream_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamException(message="Stream exception") + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream exception") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_not_found_exception(self): + timeline = TwitterTimelineFactory.create() + + self.mocked_read.side_effect = StreamNotFoundException( + message="Stream not found" + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream not found") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_denied_exception(self): + user = UserFactory( + twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4()) + ) + timeline = TwitterTimelineFactory(user=user) + + self.mocked_read.side_effect = StreamDeniedException(message="Token expired") + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Token expired") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + user.refresh_from_db() + timeline.refresh_from_db() + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) + + def test_client_catches_stream_timed_out_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamTimeOutException( + message="Stream timed out" + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream timed out") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_too_many_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamTooManyException + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Too many requests") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_stream_parse_exception(self): + timeline = TwitterTimelineFactory() + + self.mocked_read.side_effect = StreamParseException( + message="Stream could not be parsed" + ) + + with TwitterClient([timeline]) as client: + for data, stream in client: + with self.subTest(data=data, stream=stream): + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(stream.rule.error, "Stream could not be parsed") + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() + + def test_client_catches_long_exception_text(self): + timeline = TwitterTimelineFactory() + mock_stream = Mock(rule=timeline) + + self.mocked_read.side_effect = StreamParseException(message=words(1000)) + + with TwitterClient([timeline]) as client: + for data, stream in client: + self.assertIsNone(data) + self.assertIsNone(stream) + self.assertEquals(len(stream.rule.error), 1024) + self.assertEquals(stream.rule.succeeded, False) + + self.mocked_read.assert_called() diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index b503636..7e09c69 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -203,7 +203,9 @@ class TwitterClient(PostClient): break except StreamDeniedException as e: - logger.warning(f"Access token expired for user {stream.user.pk}") + logger.warning( + f"Access token expired for user {stream.rule.user.pk}" + ) stream.rule.user.twitter_oauth_token = None stream.rule.user.twitter_oauth_token_secret = None From a1ebbe68f5b9e1eb53f9fc7caee360eef41345c5 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 22:35:45 +0200 Subject: [PATCH 61/69] Add TwitterCollector tests --- .../tests/twitter/collector/mocks.py | 227 ++++++++++++++++++ .../tests/twitter/collector/tests.py | 180 ++++++++++++++ src/newsreader/news/collection/twitter.py | 2 +- 3 files changed, 408 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/tests/twitter/collector/mocks.py b/src/newsreader/news/collection/tests/twitter/collector/mocks.py index e69de29..c57f9cf 100644 --- a/src/newsreader/news/collection/tests/twitter/collector/mocks.py +++ b/src/newsreader/news/collection/tests/twitter/collector/mocks.py @@ -0,0 +1,227 @@ +# retrieved with: +# curl -X GET -H "Authorization: Bearer " "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys + +simple_mock = [ + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 20:32:22 +0000 2020", + "display_text_range": [0, 111], + "entities": { + "hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}], + "symbols": [], + "urls": [], + "user_mentions": [], + }, + "favorite_count": 54, + "favorited": False, + "full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?", + "geo": None, + "id": 1307054882210435074, + "id_str": "1307054882210435074", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "retweet_count": 9, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, + { + "contributors": None, + "coordinates": None, + "created_at": "Fri Sep 18 18:50:11 +0000 2020", + "display_text_range": [0, 271], + "entities": { + "hashtags": [{"indices": [211, 218], "text": "Twitch"}], + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ], + "symbols": [], + "urls": [ + { + "display_url": "twitch.tv/starcitizen", + "expanded_url": "http://twitch.tv/starcitizen", + "indices": [248, 271], + "url": "https://t.co/2AdNovhpFW", + } + ], + "user_mentions": [], + }, + "extended_entities": { + "media": [ + { + "display_url": "pic.twitter.com/Cey5JpR1i9", + "expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1", + "id": 1307028141697765376, + "id_str": "1307028141697765376", + "indices": [272, 295], + "media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg", + "sizes": { + "large": {"h": 1090, "resize": "fit", "w": 1920}, + "medium": {"h": 681, "resize": "fit", "w": 1200}, + "small": {"h": 386, "resize": "fit", "w": 680}, + "thumb": {"h": 150, "resize": "crop", "w": 150}, + }, + "type": "photo", + "url": "https://t.co/Cey5JpR1i9", + } + ] + }, + "favorite_count": 90, + "favorited": False, + "full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9", + "geo": None, + "id": 1307029168941461504, + "id_str": "1307029168941461504", + "in_reply_to_screen_name": None, + "in_reply_to_status_id": None, + "in_reply_to_status_id_str": None, + "in_reply_to_user_id": None, + "in_reply_to_user_id_str": None, + "is_quote_status": False, + "lang": "en", + "place": None, + "possibly_sensitive": False, + "retweet_count": 13, + "retweeted": False, + "source": 'Twitter Web App', + "truncated": False, + "user": { + "contributors_enabled": False, + "created_at": "Wed Sep 05 00:58:11 +0000 2012", + "default_profile": False, + "default_profile_image": False, + "description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.", + "entities": { + "description": {"urls": []}, + "url": { + "urls": [ + { + "display_url": "robertsspaceindustries.com", + "expanded_url": "http://www.robertsspaceindustries.com", + "indices": [0, 23], + "url": "https://t.co/iqO6apof3y", + } + ] + }, + }, + "favourites_count": 4831, + "follow_request_sent": None, + "followers_count": 106971, + "following": None, + "friends_count": 204, + "geo_enabled": False, + "has_extended_profile": False, + "id": 803542770, + "id_str": "803542770", + "is_translation_enabled": False, + "is_translator": False, + "lang": None, + "listed_count": 893, + "location": "Roberts Space Industries", + "name": "Star Citizen", + "notifications": None, + "profile_background_color": "131516", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", + "profile_background_tile": False, + "profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186", + "profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg", + "profile_link_color": "0A5485", + "profile_sidebar_border_color": "FFFFFF", + "profile_sidebar_fill_color": "EFEFEF", + "profile_text_color": "333333", + "profile_use_background_image": True, + "protected": False, + "screen_name": "RobertsSpaceInd", + "statuses_count": 6368, + "time_zone": None, + "translator_type": "none", + "url": "https://t.co/iqO6apof3y", + "utc_offset": None, + "verified": True, + }, + }, +] + +empty_mock = [] diff --git a/src/newsreader/news/collection/tests/twitter/collector/tests.py b/src/newsreader/news/collection/tests/twitter/collector/tests.py index e69de29..cf4e3ea 100644 --- a/src/newsreader/news/collection/tests/twitter/collector/tests.py +++ b/src/newsreader/news/collection/tests/twitter/collector/tests.py @@ -0,0 +1,180 @@ +from datetime import datetime +from unittest.mock import patch +from uuid import uuid4 + +from django.test import TestCase +from django.utils import timezone + +import pytz + +from freezegun import freeze_time +from ftfy import fix_text + +from newsreader.news.collection.choices import RuleTypeChoices +from newsreader.news.collection.exceptions import ( + StreamDeniedException, + StreamForbiddenException, + StreamNotFoundException, + StreamTimeOutException, +) +from newsreader.news.collection.tests.factories import TwitterTimelineFactory +from newsreader.news.collection.tests.twitter.collector.mocks import ( + empty_mock, + simple_mock, +) +from newsreader.news.collection.twitter import TWITTER_URL, TwitterCollector +from newsreader.news.collection.utils import truncate_text +from newsreader.news.core.models import Post + + +@freeze_time("2020-09-26 14:40:00") +class TwitterCollectorTestCase(TestCase): + def setUp(self): + patched_get = patch("newsreader.news.collection.twitter.fetch") + self.mocked_fetch = patched_get.start() + + patched_parse = patch("newsreader.news.collection.twitter.TwitterStream.parse") + self.mocked_parse = patched_parse.start() + + def tearDown(self): + patch.stopall() + + def test_simple_batch(self): + self.mocked_parse.return_value = simple_mock + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + screen_name="RobertsSpaceInd", + enabled=True, + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertCountEqual( + Post.objects.values_list("remote_identifier", flat=True), + ("1307054882210435074", "1307029168941461504"), + ) + + self.assertEquals(timeline.succeeded, True) + self.assertEquals(timeline.last_run, timezone.now()) + self.assertIsNone(timeline.error) + + post = Post.objects.get( + remote_identifier="1307054882210435074", + rule__type=RuleTypeChoices.twitter_timeline, + ) + + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 20, 32, 22)) + ) + + title = truncate_text( + Post, + "title", + "It's a close match-up for #SCShipShowdown today! Which Aegis ship" + " do you think will make it to the Semi-Finals?", + ) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals(post.title, title) + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307054882210435074" + ) + + post = Post.objects.get( + remote_identifier="1307029168941461504", + rule__type=RuleTypeChoices.twitter_timeline, + ) + + self.assertEquals( + post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 18, 50, 11)) + ) + + body = fix_text( + "We\u2019re welcoming members of our Builds, Publishes and Platform" + " teams on Star Citizen Live to talk about the process involved in" + " bringing everyone\u2019s work together and getting it out into your" + " hands. Going live on #Twitch in 10 minutes." + " \ud83c\udfa5\ud83d\udd34 \n\nTune in:" + " https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9" + ) + + title = truncate_text(Post, "title", body) + + self.assertEquals(post.author, "RobertsSpaceInd") + self.assertEquals(post.title, title) + self.assertEquals( + post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307029168941461504" + ) + + def test_empty_batch(self): + self.mocked_parse.return_value = empty_mock + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + + self.assertEquals(timeline.succeeded, True) + self.assertEquals(timeline.last_run, timezone.now()) + self.assertIsNone(timeline.error) + + def test_not_found(self): + self.mocked_fetch.side_effect = StreamNotFoundException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream not found") + + def test_denied(self): + self.mocked_fetch.side_effect = StreamDeniedException + + timeline = TwitterTimelineFactory( + user__twitter_oauth_token=str(uuid4()), + user__twitter_oauth_token_secret=str(uuid4()), + ) + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream does not have sufficient permissions") + + user = timeline.user + + self.assertIsNone(user.twitter_oauth_token) + self.assertIsNone(user.twitter_oauth_token_secret) + + def test_forbidden(self): + self.mocked_fetch.side_effect = StreamForbiddenException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream forbidden") + + def test_timed_out(self): + self.mocked_fetch.side_effect = StreamTimeOutException + + timeline = TwitterTimelineFactory() + + collector = TwitterCollector() + collector.collect(rules=[timeline]) + + self.assertEquals(Post.objects.count(), 0) + self.assertEquals(timeline.succeeded, False) + self.assertEquals(timeline.error, "Stream timed out") diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 7e09c69..0824532 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -154,7 +154,7 @@ class TwitterBuilder(PostBuilder): class TwitterStream(PostStream): - rule_type = RuleTypeChoices.subreddit + rule_type = RuleTypeChoices.twitter_timeline def read(self): oauth = OAuth( From b53c40a3ddda1e6cfde3caad8af7d59e43db6047 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 22:45:04 +0200 Subject: [PATCH 62/69] Add twitter timeline task --- src/newsreader/news/collection/tasks.py | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/newsreader/news/collection/tasks.py b/src/newsreader/news/collection/tasks.py index a04c5f9..926b05b 100644 --- a/src/newsreader/news/collection/tasks.py +++ b/src/newsreader/news/collection/tasks.py @@ -114,6 +114,40 @@ class RedditTokenTask(app.Task): user.save() +class TwitterTimelineTask(app.Task): + name = "TwitterTimelineTask" + ignore_result = True + + def run(self, user_pk): + from newsreader.news.collection.twitter import ( + TwitterCollector, + TwitterTimeLineScheduler, + ) + + try: + user = User.objects.get(pk=user_pk) + except ObjectDoesNotExist: + message = f"User {user_pk} does not exist" + logger.exception(message) + + raise Reject(reason=message, requeue=False) + + with MemCacheLock("f{user.email}-timeline-task", self.app.oid) as acquired: + if acquired: + logger.info(f"Running twitter timeline task for user {user_pk}") + + scheduler = TwitterTimeLineScheduler(user) + timelines = scheduler.get_scheduled_rules() + + collector = TwitterCollector() + collector.collect(rules=timelines) + else: + logger.warning(f"Cancelling task due to existing lock") + + raise Reject(reason="Task already running", requeue=False) + + FeedTask = app.register_task(FeedTask()) RedditTask = app.register_task(RedditTask()) RedditTokenTask = app.register_task(RedditTokenTask()) +TwitterTimelineTask = app.register_task(TwitterTimelineTask()) From 96b8ea7db31032c336ec57557304c90daf556b1a Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 23:08:40 +0200 Subject: [PATCH 63/69] Use correct api urls for fetching timelines --- src/newsreader/news/collection/forms/twitter.py | 4 ++-- .../collection/tests/views/test_twitter_views.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/newsreader/news/collection/forms/twitter.py b/src/newsreader/news/collection/forms/twitter.py index 5603c7a..902652b 100644 --- a/src/newsreader/news/collection/forms/twitter.py +++ b/src/newsreader/news/collection/forms/twitter.py @@ -6,7 +6,7 @@ import pytz from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.forms.base import CollectionRuleForm from newsreader.news.collection.models import CollectionRule -from newsreader.news.collection.twitter import TWITTER_URL +from newsreader.news.collection.twitter import TWITTER_API_URL class TwitterTimelineForm(CollectionRuleForm): @@ -22,7 +22,7 @@ class TwitterTimelineForm(CollectionRuleForm): instance.type = RuleTypeChoices.twitter_timeline instance.timezone = str(pytz.utc) - instance.url = f"{TWITTER_URL}/{instance.screen_name}" + instance.url = f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name={instance.screen_name}&tweet_mode=extended" if commit: instance.save() diff --git a/src/newsreader/news/collection/tests/views/test_twitter_views.py b/src/newsreader/news/collection/tests/views/test_twitter_views.py index 129ed79..d9afa26 100644 --- a/src/newsreader/news/collection/tests/views/test_twitter_views.py +++ b/src/newsreader/news/collection/tests/views/test_twitter_views.py @@ -9,7 +9,7 @@ from newsreader.news.collection.choices import RuleTypeChoices from newsreader.news.collection.models import CollectionRule from newsreader.news.collection.tests.factories import TwitterTimelineFactory from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCase -from newsreader.news.collection.twitter import TWITTER_URL +from newsreader.news.collection.twitter import TWITTER_API_URL from newsreader.news.core.tests.factories import CategoryFactory @@ -33,7 +33,10 @@ class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase): rule = CollectionRule.objects.get(name="new rule") self.assertEquals(rule.type, RuleTypeChoices.twitter_timeline) - self.assertEquals(rule.url, f"{TWITTER_URL}/RobertsSpaceInd") + self.assertEquals( + rule.url, + f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended", + ) self.assertEquals(rule.timezone, str(pytz.utc)) self.assertEquals(rule.favicon, None) self.assertEquals(rule.category.pk, self.category.pk) @@ -116,7 +119,10 @@ class TwitterTimelineUpdateViewTestCase(CollectionRuleViewTestCase, TestCase): self.rule.refresh_from_db() self.assertEquals(self.rule.type, RuleTypeChoices.twitter_timeline) - self.assertEquals(self.rule.url, f"{TWITTER_URL}/CyberpunkGame") + self.assertEquals( + self.rule.url, + f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name=CyberpunkGame&tweet_mode=extended", + ) self.assertEquals(self.rule.timezone, str(pytz.utc)) self.assertEquals(self.rule.favicon, None) self.assertEquals(self.rule.category.pk, self.category.pk) From 08e0802ca4512a62cb6a040fd21b7e6c0093ce57 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sat, 26 Sep 2020 23:17:11 +0200 Subject: [PATCH 64/69] Don't create new tasks when it is disabled --- src/newsreader/news/collection/views/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/views/base.py b/src/newsreader/news/collection/views/base.py index 8a9207d..d7a3a4d 100644 --- a/src/newsreader/news/collection/views/base.py +++ b/src/newsreader/news/collection/views/base.py @@ -50,10 +50,10 @@ class TaskCreationMixin: PeriodicTask.objects.get_or_create( name=f"{self.request.user.email}-{self.task_name}", task=self.task_type, - enabled=True, defaults={ "args": json.dumps([self.request.user.pk]), "interval": task_interval, + "enabled": True, }, ) From c41f35917eb358fceb412124bfb8b839582ebbf6 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 27 Sep 2020 11:17:31 +0200 Subject: [PATCH 65/69] Skip existing posts --- .../collection/tests/twitter/builder/tests.py | 16 ++++++++++++++++ src/newsreader/news/collection/twitter.py | 5 ++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index 2943f48..f9052f9 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -22,6 +22,7 @@ from newsreader.news.collection.tests.twitter.builder.mocks import ( from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder from newsreader.news.collection.utils import truncate_text from newsreader.news.core.models import Post +from newsreader.news.core.tests.factories import PostFactory class TwitterBuilderTestCase(TestCase): @@ -394,3 +395,18 @@ class TwitterBuilderTestCase(TestCase): ), ) self.assertEquals(post.body, mark_safe(full_text)) + + def test_existing_posts(self): + builder = TwitterBuilder + + profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd") + mock_stream = Mock(rule=profile) + + PostFactory(rule=profile, remote_identifier="1291528756373286914") + PostFactory(rule=profile, remote_identifier="1288550304095416320") + + with builder(simple_mock, mock_stream) as builder: + builder.build() + builder.save() + + self.assertEquals(Post.objects.count(), 2) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 0824532..730f12e 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -52,8 +52,11 @@ class TwitterBuilder(PostBuilder): for post in self.payload: remote_identifier = post["id_str"] - url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" + if remote_identifier in self.existing_posts: + continue + + url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" body = urlize(post["full_text"], nofollow=True) title = truncate_text( Post, "title", self.sanitize_fragment(post["full_text"]) From 717191617bea143b47c76cf00e1af2da4bcf5261 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 27 Sep 2020 11:42:06 +0200 Subject: [PATCH 66/69] Use correct tweet url --- .../news/collection/tests/twitter/builder/tests.py | 8 ++++---- .../news/collection/tests/twitter/collector/tests.py | 4 ++-- src/newsreader/news/collection/twitter.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/newsreader/news/collection/tests/twitter/builder/tests.py b/src/newsreader/news/collection/tests/twitter/builder/tests.py index f9052f9..37d7ad7 100644 --- a/src/newsreader/news/collection/tests/twitter/builder/tests.py +++ b/src/newsreader/news/collection/tests/twitter/builder/tests.py @@ -65,7 +65,7 @@ class TwitterBuilderTestCase(TestCase): self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( - post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291528756373286914" + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1291528756373286914" ) self.assertEquals( post.publication_date, pytz.utc.localize(datetime(2020, 8, 7, 0, 17, 5)) @@ -81,7 +81,7 @@ class TwitterBuilderTestCase(TestCase): self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( - post.url, f"{TWITTER_URL}/RobertsSpaceInd/1288550304095416320" + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1288550304095416320" ) self.assertEquals( post.publication_date, pytz.utc.localize(datetime(2020, 7, 29, 19, 1, 47)) @@ -110,7 +110,7 @@ class TwitterBuilderTestCase(TestCase): self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( - post.url, f"{TWITTER_URL}/RobertsSpaceInd/1269039237166321664" + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1269039237166321664" ) self.assertEquals( post.publication_date, pytz.utc.localize(datetime(2020, 6, 5, 22, 51, 46)) @@ -175,7 +175,7 @@ class TwitterBuilderTestCase(TestCase): self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals( - post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291080532361527296" + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1291080532361527296" ) self.assertEquals( post.publication_date, pytz.utc.localize(datetime(2020, 8, 5, 18, 36, 0)) diff --git a/src/newsreader/news/collection/tests/twitter/collector/tests.py b/src/newsreader/news/collection/tests/twitter/collector/tests.py index cf4e3ea..766e971 100644 --- a/src/newsreader/news/collection/tests/twitter/collector/tests.py +++ b/src/newsreader/news/collection/tests/twitter/collector/tests.py @@ -80,7 +80,7 @@ class TwitterCollectorTestCase(TestCase): self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals(post.title, title) self.assertEquals( - post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307054882210435074" + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1307054882210435074" ) post = Post.objects.get( @@ -106,7 +106,7 @@ class TwitterCollectorTestCase(TestCase): self.assertEquals(post.author, "RobertsSpaceInd") self.assertEquals(post.title, title) self.assertEquals( - post.url, f"{TWITTER_URL}/RobertsSpaceInd/1307029168941461504" + post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1307029168941461504" ) def test_empty_batch(self): diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index 730f12e..e7426c4 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -56,7 +56,7 @@ class TwitterBuilder(PostBuilder): if remote_identifier in self.existing_posts: continue - url = f"{TWITTER_URL}/{rule.screen_name}/{remote_identifier}" + url = f"{TWITTER_URL}/{rule.screen_name}/status/{remote_identifier}" body = urlize(post["full_text"], nofollow=True) title = truncate_text( Post, "title", self.sanitize_fragment(post["full_text"]) From 655acaa5658e3ea085ae63ec7862184e1b8ea305 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 27 Sep 2020 15:45:23 +0200 Subject: [PATCH 67/69] Fix quoted/retweeted tweet not showing properly --- src/newsreader/news/collection/twitter.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/newsreader/news/collection/twitter.py b/src/newsreader/news/collection/twitter.py index e7426c4..dc32ecc 100644 --- a/src/newsreader/news/collection/twitter.py +++ b/src/newsreader/news/collection/twitter.py @@ -75,22 +75,12 @@ class TwitterBuilder(PostBuilder): if "retweeted_status" in post: original_post = post["retweeted_status"] - body += urlize( - format_html( - "Original tweet: {original_post}", - original_post=urlize(original_post["full_text"], nofollow=True), - ), - nofollow=True, - ) + original_tweet = urlize(original_post["full_text"], nofollow=True) + body = f"{body}
    Original tweet: {original_tweet}
    " if "quoted_status" in post: original_post = post["quoted_status"] - body += urlize( - format_html( - "Quoted tweet: {original_post}", - original_post=original_post["full_text"], - ), - nofollow=True, - ) + original_tweet = urlize(original_post["full_text"], nofollow=True) + body = f"{body}
    Quoted tweet: {original_tweet}
    " body = self.sanitize_fragment(body) From 06e104604483bb8db726e067a3b4f4f61d63cff4 Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 27 Sep 2020 15:48:58 +0200 Subject: [PATCH 68/69] Add production settings for twitter client credentials --- src/newsreader/conf/production.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/newsreader/conf/production.py b/src/newsreader/conf/production.py index bfe9818..f481885 100644 --- a/src/newsreader/conf/production.py +++ b/src/newsreader/conf/production.py @@ -46,9 +46,14 @@ TEMPLATES = [ ] # Reddit integration -REDDIT_CLIENT_ID = os.environ["REDDIT_CLIENT_ID"] -REDDIT_CLIENT_SECRET = os.environ["REDDIT_CLIENT_SECRET"] -REDDIT_REDIRECT_URL = os.environ["REDDIT_CALLBACK_URL"] +REDDIT_CLIENT_ID = os.environ.get("REDDIT_CLIENT_ID", "") +REDDIT_CLIENT_SECRET = os.environ.get("REDDIT_CLIENT_SECRET", "") +REDDIT_REDIRECT_URL = os.environ.get("REDDIT_CALLBACK_URL", "") + +# Twitter integration +TWITTER_CONSUMER_ID = os.environ.get("TWITTER_CONSUMER_ID", "") +TWITTER_CONSUMER_SECRET = os.environ.get("TWITTER_CONSUMER_SECRET", "") +TWITTER_REDIRECT_URL = os.environ.get("TWITTER_REDIRECT_URL", "") # Third party settings AXES_HANDLER = "axes.handlers.database.AxesDatabaseHandler" From 9f9a62f349b6ab0283853ea83828d479d668decc Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Sun, 27 Sep 2020 15:50:39 +0200 Subject: [PATCH 69/69] Remove TODO --- src/newsreader/accounts/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/newsreader/accounts/urls.py b/src/newsreader/accounts/urls.py index 841e40a..3cdd1b1 100644 --- a/src/newsreader/accounts/urls.py +++ b/src/newsreader/accounts/urls.py @@ -71,7 +71,6 @@ urlpatterns = [ name="password-change", ), # Integrations - # TODO update reddit callback url in reddit app settings path( "settings/integrations/reddit/callback/", login_required(RedditTemplateView.as_view()),