312 lines
10 KiB
Python
312 lines
10 KiB
Python
from datetime import datetime
|
|
from unittest.mock import MagicMock
|
|
|
|
from django.test import TestCase
|
|
from django.utils.html import format_html
|
|
|
|
import pytz
|
|
|
|
from ftfy import fix_text
|
|
|
|
from newsreader.news.collection.tests.factories import TwitterProfileFactory
|
|
from newsreader.news.collection.tests.twitter.builder.mocks import (
|
|
gif_mock,
|
|
image_mock,
|
|
quoted_mock,
|
|
retweet_mock,
|
|
simple_mock,
|
|
unsanitized_mock,
|
|
video_mock,
|
|
video_without_bitrate_mock,
|
|
)
|
|
from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder
|
|
from newsreader.news.collection.utils import truncate_text
|
|
from newsreader.news.core.models import Post
|
|
|
|
|
|
class TwitterBuilderTestCase(TestCase):
|
|
def setUp(self):
|
|
self.maxDiff = None
|
|
|
|
def test_simple_post(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((simple_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(
|
|
("1291528756373286914", "1288550304095416320"), posts.keys()
|
|
)
|
|
|
|
post = posts["1291528756373286914"]
|
|
|
|
full_text = "@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX"
|
|
|
|
self.assertEquals(post.rule, profile)
|
|
self.assertEquals(post.title, truncate_text(Post, "title", full_text))
|
|
self.assertEquals(post.body, format_html(full_text))
|
|
|
|
self.assertEquals(post.author, "RobertsSpaceInd")
|
|
self.assertEquals(
|
|
post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291528756373286914"
|
|
)
|
|
self.assertEquals(
|
|
post.publication_date, pytz.utc.localize(datetime(2020, 8, 7, 0, 17, 5))
|
|
)
|
|
|
|
post = posts["1288550304095416320"]
|
|
|
|
full_text = "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing."
|
|
|
|
self.assertEquals(post.rule, profile)
|
|
self.assertEquals(post.title, truncate_text(Post, "title", full_text))
|
|
self.assertEquals(post.body, format_html(full_text))
|
|
|
|
self.assertEquals(post.author, "RobertsSpaceInd")
|
|
self.assertEquals(
|
|
post.url, f"{TWITTER_URL}/RobertsSpaceInd/1288550304095416320"
|
|
)
|
|
self.assertEquals(
|
|
post.publication_date, pytz.utc.localize(datetime(2020, 7, 29, 19, 1, 47))
|
|
)
|
|
|
|
# note that only one media type can be uploaded to an Tweet
|
|
# see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object
|
|
def test_images_in_post(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((image_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(("1269039237166321664",), posts.keys())
|
|
|
|
post = posts["1269039237166321664"]
|
|
|
|
full_text = "_ https://t.co/VjEeDrL1iA"
|
|
|
|
self.assertEquals(post.rule, profile)
|
|
self.assertEquals(post.title, full_text)
|
|
|
|
self.assertEquals(post.author, "RobertsSpaceInd")
|
|
self.assertEquals(
|
|
post.url, f"{TWITTER_URL}/RobertsSpaceInd/1269039237166321664"
|
|
)
|
|
self.assertEquals(
|
|
post.publication_date, pytz.utc.localize(datetime(2020, 6, 5, 22, 51, 46))
|
|
)
|
|
|
|
self.assertIn(full_text, post.body)
|
|
self.assertInHTML(
|
|
"""<div><img alt="1269039233072689152" src="https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg" loading="lazy"></div>""",
|
|
post.body,
|
|
count=1,
|
|
)
|
|
self.assertInHTML(
|
|
"""<div><img alt="1269039233068527618" src="https://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg" loading="lazy"></div>""",
|
|
post.body,
|
|
count=1,
|
|
)
|
|
|
|
def test_videos_in_post(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((video_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(
|
|
("1291080532361527296", "1291079386821582849"), posts.keys()
|
|
)
|
|
|
|
post = posts["1291080532361527296"]
|
|
|
|
full_text = fix_text(
|
|
"Small enough to access hard-to-reach ore deposits, but with enough"
|
|
" power to get through the tough jobs, Greycat\u2019s ROC perfectly"
|
|
" complements any mining operation. \n\nDetails:"
|
|
" https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH"
|
|
)
|
|
|
|
self.assertEquals(post.rule, profile)
|
|
self.assertEquals(post.title, truncate_text(Post, "title", full_text))
|
|
|
|
self.assertEquals(post.author, "RobertsSpaceInd")
|
|
self.assertEquals(
|
|
post.url, f"{TWITTER_URL}/RobertsSpaceInd/1291080532361527296"
|
|
)
|
|
self.assertEquals(
|
|
post.publication_date, pytz.utc.localize(datetime(2020, 8, 5, 18, 36, 0))
|
|
)
|
|
|
|
self.assertIn(full_text, post.body)
|
|
self.assertInHTML(
|
|
"""<div><video controls muted=""><source src="https://video.twimg.com/amplify_video/1291074294747770880/vid/1280x720/J05_p6q74ZUN4csg.mp4?tag=13" type="video/mp4" /></video></div>""",
|
|
post.body,
|
|
count=1,
|
|
)
|
|
|
|
def test_video_without_bitrate(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((video_without_bitrate_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(("1291080532361527296",), posts.keys())
|
|
|
|
post = posts["1291080532361527296"]
|
|
|
|
self.assertInHTML(
|
|
"""<div><video controls muted=""><source src="https://video.twimg.com/amplify_video/1291074294747770880/pl/kMYgFEoRyoW99o-i.m3u8?tag=13" type="application/x-mpegURL"></video></div>""",
|
|
post.body,
|
|
count=1,
|
|
)
|
|
|
|
def test_GIFs_in_post(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((gif_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(
|
|
("1289337776140296193", "1288965215648849920"), posts.keys()
|
|
)
|
|
|
|
post = posts["1289337776140296193"]
|
|
|
|
self.assertInHTML(
|
|
"""<div><video controls muted=""><source src="https://video.twimg.com/tweet_video/EeSl3sPUcAAyE4J.mp4" type="video/mp4"></video></div>""",
|
|
post.body,
|
|
count=1,
|
|
)
|
|
|
|
self.assertIn("@Xenosystems https://t.co/wxvioLCJ6h", post.body)
|
|
|
|
def test_retweet_post(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((retweet_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(
|
|
("1291117030486106112", "1288825524878336000"), posts.keys()
|
|
)
|
|
|
|
post = posts["1291117030486106112"]
|
|
|
|
self.assertIn(
|
|
fix_text(
|
|
"RT @Narayan_N7: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo,"
|
|
" the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPle\u2026"
|
|
),
|
|
post.body,
|
|
)
|
|
|
|
self.assertIn(
|
|
fix_text(
|
|
"Original tweet: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch"
|
|
" 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease,"
|
|
" share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe!"
|
|
" \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\nhttps://t.co/j4QahHzbw4"
|
|
),
|
|
post.body,
|
|
)
|
|
|
|
def test_quoted_post(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((quoted_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(
|
|
("1290801039075979264", "1289320160021495809"), posts.keys()
|
|
)
|
|
|
|
post = posts["1290801039075979264"]
|
|
|
|
self.assertIn(
|
|
fix_text("Bonne nuit \ud83c\udf3a\ud83d\udeeb https://t.co/WyznJwCJLp"),
|
|
post.body,
|
|
)
|
|
|
|
self.assertIn(
|
|
fix_text(
|
|
"Quoted tweet: #Starcitizen Le jeu est beau. Bonne nuit"
|
|
" @RobertsSpaceInd https://t.co/xCXun68V3r"
|
|
),
|
|
post.body,
|
|
)
|
|
|
|
def test_empty_data(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder(([], mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
self.assertEquals(Post.objects.count(), 0)
|
|
|
|
def test_html_sanitizing(self):
|
|
builder = TwitterBuilder
|
|
|
|
profile = TwitterProfileFactory(screen_name="RobertsSpaceInd")
|
|
mock_stream = MagicMock(rule=profile)
|
|
|
|
with builder((unsanitized_mock, mock_stream)) as builder:
|
|
builder.save()
|
|
|
|
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
|
|
|
self.assertCountEqual(("1291528756373286914",), posts.keys())
|
|
|
|
post = posts["1291528756373286914"]
|
|
|
|
full_text = (
|
|
"@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX"
|
|
" <article></article>"
|
|
)
|
|
|
|
self.assertEquals(post.rule, profile)
|
|
self.assertEquals(post.title, truncate_text(Post, "title", full_text))
|
|
self.assertEquals(post.body, format_html(full_text))
|
|
|
|
self.assertInHTML("<script></script>", post.body, count=0)
|
|
self.assertInHTML("<article></article>", post.body, count=1)
|
|
|
|
self.assertInHTML("<script></script>", post.title, count=0)
|
|
self.assertInHTML("<article></article>", post.title, count=1)
|