sonny/newsreader
1
0
Fork 0
Code Issues 13 Pull requests 2 Projects Releases 58 Packages Wiki Activity Actions

Show direct url when no body was found

Browse source
This commit is contained in:
Sonny 2020-07-19 22:59:39 +02:00
parent 0579d6f61b
commit b3adab82fa
2 changed files with 38 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/newsreader/news/collection/reddit.py
View file

@ -45,6 +45,9 @@ RATE_LIMIT_DURATION = timedelta(seconds=60)
REDDIT_IMAGE_EXTENSIONS = (".jpg", ".png", ".gif") REDDIT_IMAGE_EXTENSIONS = (".jpg", ".png", ".gif")
REDDIT_VIDEO_EXTENSIONS = (".mp4", ".gifv", ".webm") REDDIT_VIDEO_EXTENSIONS = (".mp4", ".gifv", ".webm")
# see type prefixes on https://www.reddit.com/dev/api/
REDDIT_POST = "t3"
def get_reddit_authorization_url(user): def get_reddit_authorization_url(user):
state = str(uuid4()) state = str(uuid4())
@ -117,20 +120,23 @@ class RedditBuilder(Builder):
results = {} results = {}
for post in posts: for post in posts:
if not "data" in post: if not "data" in post or post["kind"] != REDDIT_POST:
continue continue
remote_identifier = post["data"]["id"] data = post["data"]
title = truncate_text(Post, "title", post["data"]["title"])
author = truncate_text(Post, "author", post["data"]["author"]) remote_identifier = data["id"]
post_url_fragment = post["data"]["permalink"] title = truncate_text(Post, "title", data["title"])
direct_url = post["data"]["url"] author = truncate_text(Post, "author", data["author"])
is_text_post = post["data"]["is_self"] post_url_fragment = data["permalink"]
direct_url = data["url"]
is_text_post = data["is_self"]
if remote_identifier in results: if remote_identifier in results:
continue continue
uncleaned_body = post["data"]["selftext_html"] if is_text_post:
uncleaned_body = data["selftext_html"]
unescaped_body = unescape(uncleaned_body) if uncleaned_body else "" unescaped_body = unescape(uncleaned_body) if uncleaned_body else ""
body = ( body = (
bleach.clean( bleach.clean(
@ -143,14 +149,13 @@ class RedditBuilder(Builder):
if unescaped_body if unescaped_body
else "" else ""
) )
elif direct_url.endswith(REDDIT_IMAGE_EXTENSIONS):
if not is_text_post and direct_url.endswith(REDDIT_IMAGE_EXTENSIONS):
body = f"<div><img alt='{title}' src='{direct_url}' loading='lazy' /></div>" body = f"<div><img alt='{title}' src='{direct_url}' loading='lazy' /></div>"
elif not is_text_post and post["data"]["is_video"]: elif data["is_video"]:
video_info = post["data"]["secure_media"]["reddit_video"] video_info = data["secure_media"]["reddit_video"]
body = f"<div><video controls muted><source src='{video_info['fallback_url']}' type='video/mp4' /></video></div>" body = f"<div><video controls muted><source src='{video_info['fallback_url']}' type='video/mp4' /></video></div>"
elif not is_text_post and direct_url.endswith(REDDIT_VIDEO_EXTENSIONS): elif direct_url.endswith(REDDIT_VIDEO_EXTENSIONS):
extension = next( extension = next(
extension.replace(".", "") extension.replace(".", "")
for extension in REDDIT_VIDEO_EXTENSIONS for extension in REDDIT_VIDEO_EXTENSIONS
@ -161,6 +166,8 @@ class RedditBuilder(Builder):
body = f"<div><video controls muted><source src='{direct_url.replace(extension, 'mp4')}' type='video/mp4' /></video></div>" body = f"<div><video controls muted><source src='{direct_url.replace(extension, 'mp4')}' type='video/mp4' /></video></div>"
else: else:
body = f"<div><video controls muted><source src='{direct_url}' type='video/{extension}' /></video></div>" body = f"<div><video controls muted><source src='{direct_url}' type='video/{extension}' /></video></div>"
else:
body = f"<div><a target='_blank' rel='noopener noreferrer' alt='{title}' href='{direct_url}' class='link'>Direct url</a></div>"
try: try:
parsed_date = datetime.fromtimestamp(post["data"]["created_utc"]) parsed_date = datetime.fromtimestamp(post["data"]["created_utc"])
@ -169,7 +176,7 @@ class RedditBuilder(Builder):
logging.warning(f"Failed parsing timestamp from {url_fragment}") logging.warning(f"Failed parsing timestamp from {url_fragment}")
created_date = timezone.now() created_date = timezone.now()
data = { post_data = {
"remote_identifier": remote_identifier, "remote_identifier": remote_identifier,
"title": title, "title": title,
"body": body, "body": body,
@ -182,13 +189,13 @@ class RedditBuilder(Builder):
if remote_identifier in self.existing_posts: if remote_identifier in self.existing_posts:
existing_post = self.existing_posts[remote_identifier] existing_post = self.existing_posts[remote_identifier]
for key, value in data.items(): for key, value in post_data.items():
setattr(existing_post, key, value) setattr(existing_post, key, value)
results[existing_post.remote_identifier] = existing_post results[existing_post.remote_identifier] = existing_post
continue continue
results[remote_identifier] = Post(**data) results[remote_identifier] = Post(**post_data)
return results.values() return results.values()

4
src/newsreader/news/collection/tests/reddit/builder/tests.py
View file

@ -240,3 +240,7 @@ class RedditBuilderTestCase(TestCase):
@skip("Not implemented") @skip("Not implemented")
def test_link_only_post(self): def test_link_only_post(self):
pass pass
@skip("Not implemented")
def test_skip_not_known_post_type(self):
pass