Remove twitter integration
This commit is contained in:
parent
e09b3d6e4c
commit
b78f03d3b0
45 changed files with 27 additions and 5300 deletions
|
|
@ -33,11 +33,6 @@ x-django-env: &django-env
|
|||
REDDIT_CLIENT_SECRET:
|
||||
REDDIT_CALLBACK_URL:
|
||||
|
||||
# Twitter
|
||||
TWITTER_CONSUMER_ID:
|
||||
TWITTER_CONSUMER_SECRET:
|
||||
TWITTER_REDIRECT_URL:
|
||||
|
||||
# Sentry
|
||||
SENTRY_DSN:
|
||||
|
||||
|
|
|
|||
|
|
@ -17,12 +17,6 @@ class UserAdminForm(UserChangeForm):
|
|||
"reddit_refresh_token": forms.PasswordInput(
|
||||
attrs={"size": "90"}, render_value=True
|
||||
),
|
||||
"twitter_oauth_token": forms.PasswordInput(
|
||||
attrs={"size": "90"}, render_value=True
|
||||
),
|
||||
"twitter_oauth_token_secret": forms.PasswordInput(
|
||||
attrs={"size": "90"}, render_value=True
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -44,10 +38,6 @@ class UserAdmin(DjangoUserAdmin):
|
|||
_("Reddit settings"),
|
||||
{"fields": ("reddit_access_token", "reddit_refresh_token")},
|
||||
),
|
||||
(
|
||||
_("Twitter settings"),
|
||||
{"fields": ("twitter_oauth_token", "twitter_oauth_token_secret")},
|
||||
),
|
||||
(
|
||||
_("Permission settings"),
|
||||
{"classes": ("collapse",), "fields": ("is_staff", "is_superuser")},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
# Generated by Django 3.2.25 on 2024-09-06 07:14
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('accounts', '0016_alter_user_first_name'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='user',
|
||||
name='twitter_oauth_token',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='user',
|
||||
name='twitter_oauth_token_secret',
|
||||
),
|
||||
]
|
||||
|
|
@ -43,10 +43,6 @@ class User(AbstractUser):
|
|||
reddit_refresh_token = models.CharField(max_length=255, blank=True, null=True)
|
||||
reddit_access_token = models.CharField(max_length=255, blank=True, null=True)
|
||||
|
||||
# twitter settings
|
||||
twitter_oauth_token = models.CharField(max_length=255, blank=True, null=True)
|
||||
twitter_oauth_token_secret = models.CharField(max_length=255, blank=True, null=True)
|
||||
|
||||
# settings
|
||||
auto_mark_read = models.BooleanField(
|
||||
_("Auto read marking"),
|
||||
|
|
@ -68,7 +64,3 @@ class User(AbstractUser):
|
|||
tasks.delete()
|
||||
|
||||
return super().delete(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def has_twitter_auth(self):
|
||||
return self.twitter_oauth_token and self.twitter_oauth_token_secret
|
||||
|
|
|
|||
|
|
@ -40,31 +40,6 @@
|
|||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="integrations">
|
||||
<h3 class="integrations__title">Twitter</h3>
|
||||
<div class="integrations__controls">
|
||||
{% if twitter_auth_url %}
|
||||
<a class="link button button--twitter" href="{{ twitter_auth_url }}">
|
||||
{% trans "Authorize account" %}
|
||||
</a>
|
||||
{% else %}
|
||||
<button class="button button--twitter button--disabled" disabled>
|
||||
{% trans "Authorize account" %}
|
||||
</button>
|
||||
{% endif %}
|
||||
|
||||
{% if twitter_revoke_url %}
|
||||
<a class="link button button--twitter" href="{{ twitter_revoke_url }}">
|
||||
{% trans "Deauthorize account" %}
|
||||
</a>
|
||||
{% else %}
|
||||
<button class="button button--twitter button--disabled" disabled>
|
||||
{% trans "Deauthorize account" %}
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
{% extends "base.html" %}
|
||||
{% load i18n %}
|
||||
|
||||
{% block content %}
|
||||
<main id="twitter--page" class="main">
|
||||
<section class="section text-section">
|
||||
{% if error %}
|
||||
<h1 class="h1">{% trans "Twitter authorization failed" %}</h1>
|
||||
<p>{{ error }}</p>
|
||||
{% elif authorized %}
|
||||
<h1 class="h1">{% trans "Twitter account is linked" %}</h1>
|
||||
<p>{% trans "Your Twitter account was successfully linked." %}</p>
|
||||
{% endif %}
|
||||
|
||||
<p>
|
||||
<a class="link" href="{% url 'accounts:settings:integrations' %}">{% trans "Return to integrations page" %}</a>
|
||||
</p>
|
||||
</section>
|
||||
</main>
|
||||
{% endblock %}
|
||||
|
|
@ -14,7 +14,6 @@ from newsreader.news.collection.exceptions import (
|
|||
StreamException,
|
||||
StreamTooManyException,
|
||||
)
|
||||
from newsreader.news.collection.twitter import TWITTER_AUTH_URL
|
||||
|
||||
|
||||
class IntegrationsViewTestCase(TestCase):
|
||||
|
|
@ -275,263 +274,3 @@ class RedditRevokeRedirectViewTestCase(TestCase):
|
|||
|
||||
self.assertEquals(self.user.reddit_access_token, "jadajadajada")
|
||||
self.assertEquals(self.user.reddit_refresh_token, "jadajadajada")
|
||||
|
||||
|
||||
class TwitterRevokeRedirectView(TestCase):
|
||||
def setUp(self):
|
||||
self.user = UserFactory(email="test@test.nl", password="test")
|
||||
self.client.force_login(self.user)
|
||||
|
||||
self.patch = patch("newsreader.accounts.views.integrations.post")
|
||||
self.mocked_post = self.patch.start()
|
||||
|
||||
def tearDown(self):
|
||||
patch.stopall()
|
||||
|
||||
def test_simple(self):
|
||||
self.user.twitter_oauth_token = "jadajadajada"
|
||||
self.user.twitter_oauth_token_secret = "jadajadajada"
|
||||
self.user.save()
|
||||
|
||||
response = self.client.get(reverse("accounts:settings:twitter-revoke"))
|
||||
|
||||
self.assertRedirects(response, reverse("accounts:settings:integrations"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertIsNone(self.user.twitter_oauth_token)
|
||||
self.assertIsNone(self.user.twitter_oauth_token_secret)
|
||||
|
||||
def test_no_authorized_account(self):
|
||||
self.user.twitter_oauth_token = None
|
||||
self.user.twitter_oauth_token_secret = None
|
||||
self.user.save()
|
||||
|
||||
response = self.client.get(reverse("accounts:settings:twitter-revoke"))
|
||||
|
||||
self.assertRedirects(response, reverse("accounts:settings:integrations"))
|
||||
|
||||
self.mocked_post.assert_not_called()
|
||||
|
||||
def test_stream_exception(self):
|
||||
self.user.twitter_oauth_token = "jadajadajada"
|
||||
self.user.twitter_oauth_token_secret = "jadajadajada"
|
||||
self.user.save()
|
||||
|
||||
self.mocked_post.side_effect = StreamException
|
||||
|
||||
response = self.client.get(reverse("accounts:settings:twitter-revoke"))
|
||||
|
||||
self.assertRedirects(response, reverse("accounts:settings:integrations"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertEquals(self.user.twitter_oauth_token, "jadajadajada")
|
||||
self.assertEquals(self.user.twitter_oauth_token_secret, "jadajadajada")
|
||||
|
||||
|
||||
class TwitterAuthRedirectViewTestCase(TestCase):
|
||||
def setUp(self):
|
||||
self.user = UserFactory(email="test@test.nl", password="test")
|
||||
self.client.force_login(self.user)
|
||||
|
||||
self.patch = patch("newsreader.accounts.views.integrations.post")
|
||||
self.mocked_post = self.patch.start()
|
||||
|
||||
def tearDown(self):
|
||||
cache.clear()
|
||||
|
||||
def test_simple(self):
|
||||
self.mocked_post.return_value = Mock(
|
||||
text="oauth_token=foo&oauth_token_secret=bar"
|
||||
)
|
||||
|
||||
response = self.client.get(reverse("accounts:settings:twitter-auth"))
|
||||
|
||||
self.assertRedirects(
|
||||
response,
|
||||
f"{TWITTER_AUTH_URL}/?oauth_token=foo",
|
||||
fetch_redirect_response=False,
|
||||
)
|
||||
|
||||
cached_token = cache.get(f"twitter-{self.user.email}-token")
|
||||
cached_secret = cache.get(f"twitter-{self.user.email}-secret")
|
||||
|
||||
self.assertEquals(cached_token, "foo")
|
||||
self.assertEquals(cached_secret, "bar")
|
||||
|
||||
def test_stream_exception(self):
|
||||
self.mocked_post.side_effect = StreamException
|
||||
|
||||
response = self.client.get(reverse("accounts:settings:twitter-auth"))
|
||||
|
||||
self.assertRedirects(response, reverse("accounts:settings:integrations"))
|
||||
|
||||
cached_token = cache.get(f"twitter-{self.user.email}-token")
|
||||
cached_secret = cache.get(f"twitter-{self.user.email}-secret")
|
||||
|
||||
self.assertIsNone(cached_token)
|
||||
self.assertIsNone(cached_secret)
|
||||
|
||||
def test_unexpected_contents(self):
|
||||
self.mocked_post.return_value = Mock(text="foo=bar&oauth_token_secret=bar")
|
||||
|
||||
response = self.client.get(reverse("accounts:settings:twitter-auth"))
|
||||
|
||||
self.assertRedirects(response, reverse("accounts:settings:integrations"))
|
||||
|
||||
cached_token = cache.get(f"twitter-{self.user.email}-token")
|
||||
cached_secret = cache.get(f"twitter-{self.user.email}-secret")
|
||||
|
||||
self.assertIsNone(cached_token)
|
||||
self.assertIsNone(cached_secret)
|
||||
|
||||
|
||||
class TwitterTemplateViewTestCase(TestCase):
|
||||
def setUp(self):
|
||||
self.user = UserFactory(email="test@test.nl", password="test")
|
||||
self.client.force_login(self.user)
|
||||
|
||||
self.patch = patch("newsreader.accounts.views.integrations.post")
|
||||
self.mocked_post = self.patch.start()
|
||||
|
||||
def tearDown(self):
|
||||
cache.clear()
|
||||
|
||||
def test_simple(self):
|
||||
cache.set_many(
|
||||
{
|
||||
f"twitter-{self.user.email}-token": "foo",
|
||||
f"twitter-{self.user.email}-secret": "bar",
|
||||
}
|
||||
)
|
||||
|
||||
params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"}
|
||||
|
||||
self.mocked_post.return_value = Mock(
|
||||
text="oauth_token=realtoken&oauth_token_secret=realsecret"
|
||||
)
|
||||
|
||||
response = self.client.get(
|
||||
f"{reverse('accounts:settings:twitter-template')}?{urlencode(params)}"
|
||||
)
|
||||
|
||||
self.assertContains(response, _("Twitter account is linked"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertEquals(self.user.twitter_oauth_token, "realtoken")
|
||||
self.assertEquals(self.user.twitter_oauth_token_secret, "realsecret")
|
||||
|
||||
self.assertIsNone(cache.get(f"twitter-{self.user.email}-token"))
|
||||
self.assertIsNone(cache.get(f"twitter-{self.user.email}-secret"))
|
||||
|
||||
def test_denied(self):
|
||||
params = {"denied": "true", "oauth_token": "foo", "oauth_verifier": "barfoo"}
|
||||
|
||||
response = self.client.get(
|
||||
f"{reverse('accounts:settings:twitter-template')}?{urlencode(params)}"
|
||||
)
|
||||
|
||||
self.assertContains(response, _("Twitter authorization failed"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertIsNone(self.user.twitter_oauth_token)
|
||||
self.assertIsNone(self.user.twitter_oauth_token_secret)
|
||||
|
||||
self.mocked_post.assert_not_called()
|
||||
|
||||
def test_mismatched_token(self):
|
||||
cache.set_many(
|
||||
{
|
||||
f"twitter-{self.user.email}-token": "foo",
|
||||
f"twitter-{self.user.email}-secret": "bar",
|
||||
}
|
||||
)
|
||||
|
||||
params = {"denied": "", "oauth_token": "boo", "oauth_verifier": "barfoo"}
|
||||
|
||||
response = self.client.get(
|
||||
f"{reverse('accounts:settings:twitter-template')}?{urlencode(params)}"
|
||||
)
|
||||
|
||||
self.assertContains(response, _("OAuth tokens failed to match"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertIsNone(self.user.twitter_oauth_token)
|
||||
self.assertIsNone(self.user.twitter_oauth_token_secret)
|
||||
|
||||
self.mocked_post.assert_not_called()
|
||||
|
||||
def test_missing_secret(self):
|
||||
cache.set_many({f"twitter-{self.user.email}-token": "foo"})
|
||||
|
||||
params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"}
|
||||
|
||||
response = self.client.get(
|
||||
f"{reverse('accounts:settings:twitter-template')}?{urlencode(params)}"
|
||||
)
|
||||
|
||||
self.assertContains(response, _("No matching tokens found for this user"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertIsNone(self.user.twitter_oauth_token_secret)
|
||||
|
||||
self.mocked_post.assert_not_called()
|
||||
|
||||
def test_stream_exception(self):
|
||||
cache.set_many(
|
||||
{
|
||||
f"twitter-{self.user.email}-token": "foo",
|
||||
f"twitter-{self.user.email}-secret": "bar",
|
||||
}
|
||||
)
|
||||
|
||||
params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"}
|
||||
|
||||
self.mocked_post.side_effect = StreamException
|
||||
|
||||
response = self.client.get(
|
||||
f"{reverse('accounts:settings:twitter-template')}?{urlencode(params)}"
|
||||
)
|
||||
|
||||
self.assertContains(response, _("Failed requesting access token"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertIsNone(self.user.twitter_oauth_token)
|
||||
self.assertIsNone(self.user.twitter_oauth_token_secret)
|
||||
|
||||
self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-token"))
|
||||
self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-secret"))
|
||||
|
||||
def test_unexpected_contents(self):
|
||||
cache.set_many(
|
||||
{
|
||||
f"twitter-{self.user.email}-token": "foo",
|
||||
f"twitter-{self.user.email}-secret": "bar",
|
||||
}
|
||||
)
|
||||
|
||||
params = {"denied": "", "oauth_token": "foo", "oauth_verifier": "barfoo"}
|
||||
|
||||
self.mocked_post.return_value = Mock(
|
||||
text="foobar=boo&oauth_token_secret=realsecret"
|
||||
)
|
||||
|
||||
response = self.client.get(
|
||||
f"{reverse('accounts:settings:twitter-template')}?{urlencode(params)}"
|
||||
)
|
||||
|
||||
self.assertContains(response, _("No credentials found in Twitter response"))
|
||||
|
||||
self.user.refresh_from_db()
|
||||
|
||||
self.assertIsNone(self.user.twitter_oauth_token)
|
||||
self.assertIsNone(self.user.twitter_oauth_token_secret)
|
||||
|
||||
self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-token"))
|
||||
self.assertIsNotNone(cache.get(f"twitter-{self.user.email}-secret"))
|
||||
|
|
|
|||
|
|
@ -15,9 +15,6 @@ class UserTestCase(TestCase):
|
|||
PeriodicTask.objects.create(
|
||||
name=f"{user.email}-feed", task="FeedTask", interval=interval
|
||||
)
|
||||
PeriodicTask.objects.create(
|
||||
name=f"{user.email}-timeline", task="TwitterTimelineTask", interval=interval
|
||||
)
|
||||
|
||||
user.delete()
|
||||
|
||||
|
|
|
|||
|
|
@ -21,9 +21,6 @@ from newsreader.accounts.views import (
|
|||
RegistrationCompleteView,
|
||||
RegistrationView,
|
||||
SettingsView,
|
||||
TwitterAuthRedirectView,
|
||||
TwitterRevokeRedirectView,
|
||||
TwitterTemplateView,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -44,21 +41,6 @@ settings_patterns = [
|
|||
login_required(RedditRevokeRedirectView.as_view()),
|
||||
name="reddit-revoke",
|
||||
),
|
||||
path(
|
||||
"integrations/twitter/auth/",
|
||||
login_required(TwitterAuthRedirectView.as_view()),
|
||||
name="twitter-auth",
|
||||
),
|
||||
path(
|
||||
"integrations/twitter/callback/",
|
||||
login_required(TwitterTemplateView.as_view()),
|
||||
name="twitter-template",
|
||||
),
|
||||
path(
|
||||
"integrations/twitter/revoke/",
|
||||
login_required(TwitterRevokeRedirectView.as_view()),
|
||||
name="twitter-revoke",
|
||||
),
|
||||
path(
|
||||
"integrations/", login_required(IntegrationsView.as_view()), name="integrations"
|
||||
),
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@ from newsreader.accounts.views.integrations import (
|
|||
RedditRevokeRedirectView,
|
||||
RedditTemplateView,
|
||||
RedditTokenRedirectView,
|
||||
TwitterAuthRedirectView,
|
||||
TwitterRevokeRedirectView,
|
||||
TwitterTemplateView,
|
||||
)
|
||||
from newsreader.accounts.views.password import (
|
||||
PasswordChangeView,
|
||||
|
|
@ -34,9 +31,6 @@ __all__ = [
|
|||
"RedditRevokeRedirectView",
|
||||
"RedditTemplateView",
|
||||
"RedditTokenRedirectView",
|
||||
"TwitterAuthRedirectView",
|
||||
"TwitterRevokeRedirectView",
|
||||
"TwitterTemplateView",
|
||||
"PasswordChangeView",
|
||||
"PasswordResetCompleteView",
|
||||
"PasswordResetConfirmView",
|
||||
|
|
|
|||
|
|
@ -1,17 +1,11 @@
|
|||
import logging
|
||||
|
||||
from urllib.parse import parse_qs, urlencode
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib import messages
|
||||
from django.core.cache import cache
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import reverse_lazy
|
||||
from django.utils.translation import gettext as _
|
||||
from django.views.generic import RedirectView, TemplateView
|
||||
|
||||
from requests_oauthlib import OAuth1 as OAuth
|
||||
|
||||
from newsreader.news.collection.exceptions import StreamException
|
||||
from newsreader.news.collection.reddit import (
|
||||
get_reddit_access_token,
|
||||
|
|
@ -19,13 +13,6 @@ from newsreader.news.collection.reddit import (
|
|||
revoke_reddit_token,
|
||||
)
|
||||
from newsreader.news.collection.tasks import RedditTokenTask
|
||||
from newsreader.news.collection.twitter import (
|
||||
TWITTER_ACCESS_TOKEN_URL,
|
||||
TWITTER_AUTH_URL,
|
||||
TWITTER_REQUEST_TOKEN_URL,
|
||||
TWITTER_REVOKE_URL,
|
||||
)
|
||||
from newsreader.news.collection.utils import post
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -38,7 +25,6 @@ class IntegrationsView(TemplateView):
|
|||
return {
|
||||
**super().get_context_data(**kwargs),
|
||||
**self.get_reddit_context(**kwargs),
|
||||
**self.get_twitter_context(**kwargs),
|
||||
}
|
||||
|
||||
def get_reddit_context(self, **kwargs):
|
||||
|
|
@ -68,17 +54,6 @@ class IntegrationsView(TemplateView):
|
|||
),
|
||||
}
|
||||
|
||||
def get_twitter_context(self, **kwargs):
|
||||
twitter_revoke_url = None
|
||||
|
||||
if self.request.user.has_twitter_auth:
|
||||
twitter_revoke_url = reverse_lazy("accounts:settings:twitter-revoke")
|
||||
|
||||
return {
|
||||
"twitter_auth_url": reverse_lazy("accounts:settings:twitter-auth"),
|
||||
"twitter_revoke_url": twitter_revoke_url,
|
||||
}
|
||||
|
||||
|
||||
class RedditTemplateView(TemplateView):
|
||||
template_name = "accounts/views/reddit.html"
|
||||
|
|
@ -178,166 +153,3 @@ class RedditRevokeRedirectView(RedirectView):
|
|||
|
||||
messages.success(request, _("Reddit account deathorized"))
|
||||
return response
|
||||
|
||||
|
||||
class TwitterRevokeRedirectView(RedirectView):
|
||||
url = reverse_lazy("accounts:settings:integrations")
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
if not request.user.has_twitter_auth:
|
||||
messages.error(request, _("No twitter credentials found"))
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
oauth = OAuth(
|
||||
settings.TWITTER_CONSUMER_ID,
|
||||
client_secret=settings.TWITTER_CONSUMER_SECRET,
|
||||
resource_owner_key=request.user.twitter_oauth_token,
|
||||
resource_owner_secret=request.user.twitter_oauth_token_secret,
|
||||
)
|
||||
|
||||
try:
|
||||
post(TWITTER_REVOKE_URL, auth=oauth)
|
||||
except StreamException:
|
||||
logger.exception("Failed revoking Twitter account")
|
||||
|
||||
messages.error(request, _("Unable revoke Twitter account"))
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
request.user.twitter_oauth_token = None
|
||||
request.user.twitter_oauth_token_secret = None
|
||||
request.user.save()
|
||||
|
||||
messages.success(request, _("Twitter account revoked"))
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
|
||||
class TwitterAuthRedirectView(RedirectView):
|
||||
url = reverse_lazy("accounts:settings:integrations")
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
oauth = OAuth(
|
||||
settings.TWITTER_CONSUMER_ID,
|
||||
client_secret=settings.TWITTER_CONSUMER_SECRET,
|
||||
callback_uri=settings.TWITTER_REDIRECT_URL,
|
||||
)
|
||||
|
||||
try:
|
||||
response = post(TWITTER_REQUEST_TOKEN_URL, auth=oauth)
|
||||
except StreamException:
|
||||
logger.exception("Failed requesting Twitter authentication token")
|
||||
|
||||
messages.error(request, _("Unable to retrieve initial Twitter token"))
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
params = parse_qs(response.text)
|
||||
|
||||
try:
|
||||
request_oauth_token = params["oauth_token"][0]
|
||||
request_oauth_secret = params["oauth_token_secret"][0]
|
||||
except KeyError:
|
||||
logger.exception("No credentials found in response")
|
||||
|
||||
messages.error(request, _("Unable to retrieve initial Twitter token"))
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
cache.set_many(
|
||||
{
|
||||
f"twitter-{request.user.email}-token": request_oauth_token,
|
||||
f"twitter-{request.user.email}-secret": request_oauth_secret,
|
||||
}
|
||||
)
|
||||
|
||||
request_params = urlencode({"oauth_token": request_oauth_token})
|
||||
return redirect(f"{TWITTER_AUTH_URL}/?{request_params}")
|
||||
|
||||
|
||||
class TwitterTemplateView(TemplateView):
|
||||
template_name = "accounts/views/twitter.html"
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
context = self.get_context_data(**kwargs)
|
||||
|
||||
denied = request.GET.get("denied", False)
|
||||
oauth_token = request.GET.get("oauth_token")
|
||||
oauth_verifier = request.GET.get("oauth_verifier")
|
||||
|
||||
if denied:
|
||||
return self.render_to_response(
|
||||
{
|
||||
**context,
|
||||
"error": _("Twitter authorization failed"),
|
||||
"authorized": False,
|
||||
}
|
||||
)
|
||||
|
||||
cached_token = cache.get(f"twitter-{request.user.email}-token")
|
||||
|
||||
if oauth_token != cached_token:
|
||||
return self.render_to_response(
|
||||
{
|
||||
**context,
|
||||
"error": _("OAuth tokens failed to match"),
|
||||
"authorized": False,
|
||||
}
|
||||
)
|
||||
|
||||
cached_secret = cache.get(f"twitter-{request.user.email}-secret")
|
||||
|
||||
if not cached_token or not cached_secret:
|
||||
return self.render_to_response(
|
||||
{
|
||||
**context,
|
||||
"error": _("No matching tokens found for this user"),
|
||||
"authorized": False,
|
||||
}
|
||||
)
|
||||
|
||||
oauth = OAuth(
|
||||
settings.TWITTER_CONSUMER_ID,
|
||||
client_secret=settings.TWITTER_CONSUMER_SECRET,
|
||||
resource_owner_key=cached_token,
|
||||
resource_owner_secret=cached_secret,
|
||||
verifier=oauth_verifier,
|
||||
)
|
||||
|
||||
try:
|
||||
response = post(TWITTER_ACCESS_TOKEN_URL, auth=oauth)
|
||||
except StreamException:
|
||||
logger.exception("Failed requesting Twitter access token")
|
||||
|
||||
return self.render_to_response(
|
||||
{
|
||||
**context,
|
||||
"error": _("Failed requesting access token"),
|
||||
"authorized": False,
|
||||
}
|
||||
)
|
||||
|
||||
params = parse_qs(response.text)
|
||||
|
||||
try:
|
||||
oauth_token = params["oauth_token"][0]
|
||||
oauth_secret = params["oauth_token_secret"][0]
|
||||
except KeyError:
|
||||
logger.exception("No credentials in Twitter response")
|
||||
|
||||
return self.render_to_response(
|
||||
{
|
||||
**context,
|
||||
"error": _("No credentials found in Twitter response"),
|
||||
"authorized": False,
|
||||
}
|
||||
)
|
||||
|
||||
request.user.twitter_oauth_token = oauth_token
|
||||
request.user.twitter_oauth_token_secret = oauth_secret
|
||||
request.user.save()
|
||||
|
||||
cache.delete_many(
|
||||
[
|
||||
f"twitter-{request.user.email}-token",
|
||||
f"twitter-{request.user.email}-secret",
|
||||
]
|
||||
)
|
||||
|
||||
return self.render_to_response({**context, "error": None, "authorized": True})
|
||||
|
|
|
|||
|
|
@ -218,11 +218,7 @@ REDDIT_REDIRECT_URL = (
|
|||
)
|
||||
|
||||
# Twitter integration
|
||||
TWITTER_CONSUMER_ID = "CONSUMER_ID"
|
||||
TWITTER_CONSUMER_SECRET = "CONSUMER_SECRET"
|
||||
TWITTER_REDIRECT_URL = (
|
||||
"http://127.0.0.1:8000/accounts/settings/integrations/twitter/callback/"
|
||||
)
|
||||
TWITTER_URL = "https://twitter.com"
|
||||
|
||||
# Third party settings
|
||||
AXES_HANDLER = "axes.handlers.cache.AxesCacheHandler"
|
||||
|
|
|
|||
|
|
@ -54,11 +54,6 @@ REDDIT_CLIENT_ID = os.environ.get("REDDIT_CLIENT_ID", "")
|
|||
REDDIT_CLIENT_SECRET = os.environ.get("REDDIT_CLIENT_SECRET", "")
|
||||
REDDIT_REDIRECT_URL = os.environ.get("REDDIT_CALLBACK_URL", "")
|
||||
|
||||
# Twitter integration
|
||||
TWITTER_CONSUMER_ID = os.environ.get("TWITTER_CONSUMER_ID", "")
|
||||
TWITTER_CONSUMER_SECRET = os.environ.get("TWITTER_CONSUMER_SECRET", "")
|
||||
TWITTER_REDIRECT_URL = os.environ.get("TWITTER_REDIRECT_URL", "")
|
||||
|
||||
# Third party settings
|
||||
AXES_HANDLER = "axes.handlers.database.AxesDatabaseHandler"
|
||||
|
||||
|
|
|
|||
|
|
@ -994,8 +994,6 @@
|
|||
"email": "sonnyba871@gmail.com",
|
||||
"reddit_refresh_token": null,
|
||||
"reddit_access_token": null,
|
||||
"twitter_oauth_token": null,
|
||||
"twitter_oauth_token_secret": null,
|
||||
"auto_mark_read": true,
|
||||
"groups": [],
|
||||
"user_permissions": []
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ class PostModal extends React.Component {
|
|||
ruleUrl = `${this.props.subredditUrl}/${this.props.rule.id}/`;
|
||||
break;
|
||||
case TWITTER_TIMELINE:
|
||||
ruleUrl = `${this.props.timelineUrl}/${this.props.rule.id}/`;
|
||||
ruleUrl = '#';
|
||||
break;
|
||||
default:
|
||||
ruleUrl = `${this.props.feedUrl}/${this.props.rule.id}/`;
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
from newsreader.news.collection.forms.feed import FeedForm, OPMLImportForm
|
||||
from newsreader.news.collection.forms.reddit import SubRedditForm
|
||||
from newsreader.news.collection.forms.rules import CollectionRuleBulkForm
|
||||
from newsreader.news.collection.forms.twitter import TwitterTimelineForm
|
||||
|
||||
__all__ = [
|
||||
"FeedForm",
|
||||
"OPMLImportForm",
|
||||
"SubRedditForm",
|
||||
"CollectionRuleBulkForm",
|
||||
"TwitterTimelineForm",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
from django import forms
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
import pytz
|
||||
|
||||
from newsreader.news.collection.choices import RuleTypeChoices
|
||||
from newsreader.news.collection.forms.base import CollectionRuleForm
|
||||
from newsreader.news.collection.models import CollectionRule
|
||||
from newsreader.news.collection.twitter import TWITTER_API_URL
|
||||
|
||||
|
||||
class TwitterTimelineForm(CollectionRuleForm):
|
||||
screen_name = forms.CharField(
|
||||
max_length=255,
|
||||
label=_("Twitter profile name"),
|
||||
help_text=_("Profile name without hashtags"),
|
||||
required=True,
|
||||
)
|
||||
|
||||
def save(self, commit=True):
|
||||
instance = super().save(commit=False)
|
||||
|
||||
instance.type = RuleTypeChoices.twitter_timeline
|
||||
instance.timezone = str(pytz.utc)
|
||||
instance.url = f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name={instance.screen_name}&tweet_mode=extended"
|
||||
|
||||
if commit:
|
||||
instance.save()
|
||||
self.save_m2m()
|
||||
|
||||
return instance
|
||||
|
||||
class Meta:
|
||||
model = CollectionRule
|
||||
fields = ("name", "screen_name", "favicon", "category")
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
from django.db import models
|
||||
from django.conf import settings
|
||||
from django.urls import reverse
|
||||
from django.utils.translation import gettext as _
|
||||
|
||||
|
|
@ -72,7 +73,7 @@ class CollectionRule(TimeStampedModel):
|
|||
_("Minimum amount of comments"), default=0
|
||||
)
|
||||
|
||||
# Twitter
|
||||
# Twitter (legacy)
|
||||
screen_name = models.CharField(max_length=255, blank=True, null=True)
|
||||
|
||||
objects = CollectionRuleQuerySet.as_manager()
|
||||
|
|
@ -85,9 +86,7 @@ class CollectionRule(TimeStampedModel):
|
|||
if self.type == RuleTypeChoices.subreddit:
|
||||
return reverse("news:collection:subreddit-update", kwargs={"pk": self.pk})
|
||||
elif self.type == RuleTypeChoices.twitter_timeline:
|
||||
return reverse(
|
||||
"news:collection:twitter-timeline-update", kwargs={"pk": self.pk}
|
||||
)
|
||||
return "#not-supported"
|
||||
|
||||
return reverse("news:collection:feed-update", kwargs={"pk": self.pk})
|
||||
|
||||
|
|
@ -98,9 +97,7 @@ class CollectionRule(TimeStampedModel):
|
|||
|
||||
return self.url.replace(REDDIT_API_URL, REDDIT_URL)
|
||||
elif self.type == RuleTypeChoices.twitter_timeline:
|
||||
from newsreader.news.collection.twitter import TWITTER_URL
|
||||
|
||||
return f"{TWITTER_URL}/{self.screen_name}"
|
||||
return f"{settings.TWITTER_URL}/{self.screen_name}"
|
||||
|
||||
return self.url
|
||||
|
||||
|
|
|
|||
|
|
@ -127,39 +127,6 @@ class RedditTokenTask(app.Task):
|
|||
user.save()
|
||||
|
||||
|
||||
class TwitterTimelineTask(app.Task):
|
||||
name = "TwitterTimelineTask"
|
||||
ignore_result = True
|
||||
|
||||
def run(self, user_pk):
|
||||
from newsreader.news.collection.twitter import (
|
||||
TwitterCollector,
|
||||
TwitterTimeLineScheduler,
|
||||
)
|
||||
|
||||
try:
|
||||
user = User.objects.get(pk=user_pk)
|
||||
except ObjectDoesNotExist:
|
||||
message = f"User {user_pk} does not exist"
|
||||
logger.exception(message)
|
||||
|
||||
raise Reject(reason=message, requeue=False)
|
||||
|
||||
with MemCacheLock("f{user.email}-timeline-task", self.app.oid) as acquired:
|
||||
if acquired:
|
||||
logger.info(f"Running twitter timeline task for user {user_pk}")
|
||||
|
||||
scheduler = TwitterTimeLineScheduler(user)
|
||||
timelines = scheduler.get_scheduled_rules()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=timelines)
|
||||
else:
|
||||
logger.warning("Cancelling task due to existing lock")
|
||||
|
||||
raise Reject(reason="Task already running", requeue=False)
|
||||
|
||||
|
||||
class FaviconTask(app.Task):
|
||||
name = "FaviconTask"
|
||||
ignore_result = True
|
||||
|
|
@ -192,9 +159,6 @@ class FaviconTask(app.Task):
|
|||
if rule.type == RuleTypeChoices.subreddit:
|
||||
rule.favicon = "https://www.reddit.com/favicon.ico"
|
||||
rule.save()
|
||||
elif rule.type == RuleTypeChoices.twitter_timeline:
|
||||
rule.favicon = "https://abs.twimg.com/favicons/favicon.ico"
|
||||
rule.save()
|
||||
else:
|
||||
logger.warning("Cancelling task due to existing lock")
|
||||
|
||||
|
|
@ -205,4 +169,3 @@ FeedTask = app.register_task(FeedTask())
|
|||
FaviconTask = app.register_task(FaviconTask())
|
||||
RedditTask = app.register_task(RedditTask())
|
||||
RedditTokenTask = app.register_task(RedditTokenTask())
|
||||
TwitterTimelineTask = app.register_task(TwitterTimelineTask())
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
<a class="link button button--confirm" href="{% url "news:collection:feed-create" %}">{% trans "Add a feed" %}</a>
|
||||
<a class="link button button--confirm" href="{% url "news:collection:import" %}">{% trans "Import feeds" %}</a>
|
||||
<a class="link button button--reddit" href="{% url "news:collection:subreddit-create" %}">{% trans "Add a subreddit" %}</a>
|
||||
<a class="link button button--twitter" href="{% url "news:collection:twitter-timeline-create" %}">{% trans "Add a Twitter profile" %}</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
{% extends "base.html" %}
|
||||
{% load static %}
|
||||
|
||||
{% block content %}
|
||||
<main id="twitter--page" class="main">
|
||||
{% url "news:collection:rules" as cancel_url %}
|
||||
{% include "components/form/form.html" with form=form title="Add a Twitter profile" cancel_url=cancel_url confirm_text="Add profile" %}
|
||||
</main>
|
||||
{% endblock %}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
{% extends "base.html" %}
|
||||
{% load static i18n %}
|
||||
|
||||
{% block content %}
|
||||
<main id="twitter--page" class="main">
|
||||
{% if timeline.error %}
|
||||
{% trans "Failed to retrieve posts" as title %}
|
||||
{% include "components/textbox/textbox.html" with title=title body=timeline.error class="text-section--error" only %}
|
||||
{% endif %}
|
||||
|
||||
{% url "news:collection:rules" as cancel_url %}
|
||||
{% include "components/form/form.html" with form=form title="Update profile" cancel_url=cancel_url confirm_text="Save profile" %}
|
||||
</main>
|
||||
{% endblock %}
|
||||
|
|
@ -28,8 +28,3 @@ class FeedFactory(CollectionRuleFactory):
|
|||
class SubredditFactory(CollectionRuleFactory):
|
||||
type = RuleTypeChoices.subreddit
|
||||
website_url = REDDIT_URL
|
||||
|
||||
|
||||
class TwitterTimelineFactory(CollectionRuleFactory):
|
||||
type = RuleTypeChoices.twitter_timeline
|
||||
screen_name = factory.Faker("user_name")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,431 +0,0 @@
|
|||
from datetime import datetime
|
||||
from unittest.mock import Mock
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils.safestring import mark_safe
|
||||
|
||||
import pytz
|
||||
|
||||
from ftfy import fix_text
|
||||
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.tests.twitter.builder.mocks import (
|
||||
broken_mock,
|
||||
gif_mock,
|
||||
image_mock,
|
||||
quoted_mock,
|
||||
retweet_mock,
|
||||
simple_mock,
|
||||
unsanitized_mock,
|
||||
video_mock,
|
||||
video_without_bitrate_mock,
|
||||
)
|
||||
from newsreader.news.collection.twitter import TWITTER_URL, TwitterBuilder
|
||||
from newsreader.news.collection.utils import truncate_text
|
||||
from newsreader.news.core.models import Post
|
||||
from newsreader.news.core.tests.factories import PostFactory
|
||||
|
||||
|
||||
class TwitterBuilderTestCase(TestCase):
|
||||
def setUp(self):
|
||||
self.maxDiff = None
|
||||
|
||||
def test_simple_post(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(simple_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(
|
||||
("1291528756373286914", "1288550304095416320"), posts.keys()
|
||||
)
|
||||
|
||||
post = posts["1291528756373286914"]
|
||||
|
||||
full_text = (
|
||||
"@ArieNeoSC Here you go, goodnight!\n\n"
|
||||
"""<a href="https://t.co/trAcIxBMlX" rel="nofollow">https://t.co/trAcIxBMlX</a>"""
|
||||
)
|
||||
|
||||
self.assertEquals(post.rule, profile)
|
||||
self.assertEquals(
|
||||
post.title,
|
||||
truncate_text(
|
||||
Post,
|
||||
"title",
|
||||
"@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX",
|
||||
),
|
||||
)
|
||||
self.assertEquals(post.body, mark_safe(full_text))
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1291528756373286914"
|
||||
)
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 8, 7, 0, 17, 5))
|
||||
)
|
||||
|
||||
post = posts["1288550304095416320"]
|
||||
|
||||
full_text = "@RelicCcb Hi Christoper, we have checked the status of your investigation and it is still ongoing."
|
||||
|
||||
self.assertEquals(post.rule, profile)
|
||||
self.assertEquals(post.title, truncate_text(Post, "title", full_text))
|
||||
self.assertEquals(post.body, mark_safe(full_text))
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1288550304095416320"
|
||||
)
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 7, 29, 19, 1, 47))
|
||||
)
|
||||
|
||||
# note that only one media type can be uploaded to an Tweet
|
||||
# see https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/extended-entities-object
|
||||
def test_images_in_post(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(image_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(("1269039237166321664",), posts.keys())
|
||||
|
||||
post = posts["1269039237166321664"]
|
||||
|
||||
self.assertEquals(post.rule, profile)
|
||||
self.assertEquals(post.title, "_ https://t.co/VjEeDrL1iA")
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1269039237166321664"
|
||||
)
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 6, 5, 22, 51, 46))
|
||||
)
|
||||
|
||||
self.assertInHTML(
|
||||
"""<a href="https://t.co/VjEeDrL1iA" rel="nofollow">https://t.co/VjEeDrL1iA</a>""",
|
||||
post.body,
|
||||
count=1,
|
||||
)
|
||||
self.assertInHTML(
|
||||
"""<div><img alt="1269039233072689152" src="https://pbs.twimg.com/media/EZyIdXVU8AACPCz.jpg" loading="lazy"></div>""",
|
||||
post.body,
|
||||
count=1,
|
||||
)
|
||||
self.assertInHTML(
|
||||
"""<div><img alt="1269039233068527618" src="https://pbs.twimg.com/media/EZyIdXUVcAI3Cju.jpg" loading="lazy"></div>""",
|
||||
post.body,
|
||||
count=1,
|
||||
)
|
||||
|
||||
def test_videos_in_post(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(video_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(
|
||||
("1291080532361527296", "1291079386821582849"), posts.keys()
|
||||
)
|
||||
|
||||
post = posts["1291080532361527296"]
|
||||
|
||||
full_text = fix_text(
|
||||
"Small enough to access hard-to-reach ore deposits, but with enough"
|
||||
" power to get through the tough jobs, Greycat\u2019s ROC perfectly"
|
||||
" complements any mining operation. \n\nDetails:"
|
||||
""" <a href="https://t.co/2aH7qdOfSk" rel="nofollow">https://t.co/2aH7qdOfSk</a>"""
|
||||
""" <a href="https://t.co/mZ8CAuq3SH" rel="nofollow">https://t.co/mZ8CAuq3SH</a>"""
|
||||
)
|
||||
|
||||
self.assertEquals(post.rule, profile)
|
||||
self.assertEquals(
|
||||
post.title,
|
||||
truncate_text(
|
||||
Post,
|
||||
"title",
|
||||
fix_text(
|
||||
"Small enough to access hard-to-reach ore deposits, but with enough"
|
||||
" power to get through the tough jobs, Greycat\u2019s ROC perfectly"
|
||||
" complements any mining operation. \n\nDetails:"
|
||||
" https://t.co/2aH7qdOfSk https://t.co/mZ8CAuq3SH"
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1291080532361527296"
|
||||
)
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 8, 5, 18, 36, 0))
|
||||
)
|
||||
|
||||
self.assertIn(full_text, post.body)
|
||||
self.assertInHTML(
|
||||
"""<div><video controls muted=""><source src="https://video.twimg.com/amplify_video/1291074294747770880/vid/1280x720/J05_p6q74ZUN4csg.mp4?tag=13" type="video/mp4" /></video></div>""",
|
||||
post.body,
|
||||
count=1,
|
||||
)
|
||||
|
||||
def test_video_without_bitrate(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(video_without_bitrate_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(("1291080532361527296",), posts.keys())
|
||||
|
||||
post = posts["1291080532361527296"]
|
||||
|
||||
self.assertInHTML(
|
||||
"""<div><video controls muted=""><source src="https://video.twimg.com/amplify_video/1291074294747770880/pl/kMYgFEoRyoW99o-i.m3u8?tag=13" type="application/x-mpegURL"></video></div>""",
|
||||
post.body,
|
||||
count=1,
|
||||
)
|
||||
|
||||
def test_GIFs_in_post(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(gif_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(
|
||||
("1289337776140296193", "1288965215648849920"), posts.keys()
|
||||
)
|
||||
|
||||
post = posts["1289337776140296193"]
|
||||
|
||||
self.assertInHTML(
|
||||
"""<div><video controls muted=""><source src="https://video.twimg.com/tweet_video/EeSl3sPUcAAyE4J.mp4" type="video/mp4"></video></div>""",
|
||||
post.body,
|
||||
count=1,
|
||||
)
|
||||
|
||||
self.assertIn(
|
||||
"""@Xenosystems <a href="https://t.co/wxvioLCJ6h" rel="nofollow">https://t.co/wxvioLCJ6h</a>""",
|
||||
post.body,
|
||||
)
|
||||
|
||||
def test_retweet_post(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(retweet_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(
|
||||
("1291117030486106112", "1288825524878336000"), posts.keys()
|
||||
)
|
||||
|
||||
post = posts["1291117030486106112"]
|
||||
|
||||
self.assertIn(
|
||||
fix_text(
|
||||
"RT @Narayan_N7: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo,"
|
||||
" the patch 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPle\u2026"
|
||||
),
|
||||
post.body,
|
||||
)
|
||||
|
||||
self.assertIn(
|
||||
fix_text(
|
||||
"Original tweet: New video! #StarCitizen 3.9 vs. 3.10 comparison!\nSo, the patch"
|
||||
" 3.10 came out, which brought us quite a lot of changes!\ud83d\ude42\nPlease,"
|
||||
" share it with your friends!\ud83d\ude4f\n\nEnjoy watching and stay safe!"
|
||||
" \u2764\ufe0f\u263a\ufe0f\n@RobertsSpaceInd\n\n@CloudImperium\n\n"
|
||||
"""<a href="https://t.co/j4QahHzbw4" rel="nofollow">https://t.co/j4QahHzbw4</a>"""
|
||||
),
|
||||
post.body,
|
||||
)
|
||||
|
||||
def test_quoted_post(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(quoted_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(
|
||||
("1290801039075979264", "1289320160021495809"), posts.keys()
|
||||
)
|
||||
|
||||
post = posts["1290801039075979264"]
|
||||
|
||||
self.assertIn(
|
||||
fix_text(
|
||||
"Bonne nuit \ud83c\udf3a\ud83d\udeeb"
|
||||
""" <a href="https://t.co/WyznJwCJLp" rel="nofollow">https://t.co/WyznJwCJLp</a>"""
|
||||
),
|
||||
post.body,
|
||||
)
|
||||
|
||||
self.assertIn(
|
||||
fix_text(
|
||||
"Quoted tweet: #Starcitizen Le jeu est beau. Bonne nuit"
|
||||
""" @RobertsSpaceInd <a href="https://t.co/xCXun68V3r" rel="nofollow">https://t.co/xCXun68V3r</a>"""
|
||||
),
|
||||
post.body,
|
||||
)
|
||||
|
||||
def test_empty_data(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder([], mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
|
||||
def test_html_sanitizing(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(unsanitized_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(("1291528756373286914",), posts.keys())
|
||||
|
||||
post = posts["1291528756373286914"]
|
||||
|
||||
full_text = (
|
||||
"@ArieNeoSC Here you go, goodnight!\n\n"
|
||||
"""<a href="https://t.co/trAcIxBMlX" rel="nofollow">https://t.co/trAcIxBMlX</a>"""
|
||||
" <article></article>"
|
||||
)
|
||||
|
||||
self.assertEquals(post.rule, profile)
|
||||
self.assertEquals(
|
||||
post.title,
|
||||
truncate_text(
|
||||
Post,
|
||||
"title",
|
||||
"@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX"
|
||||
" <article></article>",
|
||||
),
|
||||
)
|
||||
self.assertEquals(post.body, mark_safe(full_text))
|
||||
|
||||
self.assertInHTML("<script></script>", post.body, count=0)
|
||||
self.assertInHTML("<article></article>", post.body, count=1)
|
||||
|
||||
self.assertInHTML("<script></script>", post.title, count=0)
|
||||
self.assertInHTML("<article></article>", post.title, count=1)
|
||||
|
||||
def test_urlize_on_urls(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(simple_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
posts = {post.remote_identifier: post for post in Post.objects.all()}
|
||||
|
||||
self.assertCountEqual(
|
||||
("1291528756373286914", "1288550304095416320"), posts.keys()
|
||||
)
|
||||
|
||||
post = posts["1291528756373286914"]
|
||||
|
||||
full_text = (
|
||||
"@ArieNeoSC Here you go, goodnight!\n\n"
|
||||
"""<a href="https://t.co/trAcIxBMlX" rel="nofollow">https://t.co/trAcIxBMlX</a>"""
|
||||
)
|
||||
|
||||
self.assertEquals(post.rule, profile)
|
||||
self.assertEquals(
|
||||
post.title,
|
||||
truncate_text(
|
||||
Post,
|
||||
"title",
|
||||
"@ArieNeoSC Here you go, goodnight!\n\nhttps://t.co/trAcIxBMlX",
|
||||
),
|
||||
)
|
||||
self.assertEquals(post.body, mark_safe(full_text))
|
||||
|
||||
def test_existing_posts(self):
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
PostFactory(rule=profile, remote_identifier="1291528756373286914")
|
||||
PostFactory(rule=profile, remote_identifier="1288550304095416320")
|
||||
|
||||
with builder(simple_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
self.assertEquals(Post.objects.count(), 2)
|
||||
|
||||
def test_bad_post(self):
|
||||
"""
|
||||
Tests that the builder will ignore posts which miss data
|
||||
"""
|
||||
builder = TwitterBuilder
|
||||
|
||||
profile = TwitterTimelineFactory(screen_name="RobertsSpaceInd")
|
||||
mock_stream = Mock(rule=profile)
|
||||
|
||||
with builder(broken_mock, mock_stream) as builder:
|
||||
builder.build()
|
||||
builder.save()
|
||||
|
||||
self.assertCountEqual(
|
||||
Post.objects.values_list("remote_identifier", flat=True),
|
||||
["1288550304095416320"],
|
||||
)
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
# retrieved with:
|
||||
# curl -X GET -H "Authorization: Bearer <TOKEN>" "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys
|
||||
|
||||
simple_mock = [
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 20:32:22 +0000 2020",
|
||||
"display_text_range": [0, 111],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}],
|
||||
"symbols": [],
|
||||
"urls": [],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"favorite_count": 54,
|
||||
"favorited": False,
|
||||
"full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?",
|
||||
"geo": None,
|
||||
"id": 1307054882210435074,
|
||||
"id_str": "1307054882210435074",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"retweet_count": 9,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 18:50:11 +0000 2020",
|
||||
"display_text_range": [0, 271],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [211, 218], "text": "Twitch"}],
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
],
|
||||
"symbols": [],
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "twitch.tv/starcitizen",
|
||||
"expanded_url": "http://twitch.tv/starcitizen",
|
||||
"indices": [248, 271],
|
||||
"url": "https://t.co/2AdNovhpFW",
|
||||
}
|
||||
],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"extended_entities": {
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
]
|
||||
},
|
||||
"favorite_count": 90,
|
||||
"favorited": False,
|
||||
"full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9",
|
||||
"geo": None,
|
||||
"id": 1307029168941461504,
|
||||
"id_str": "1307029168941461504",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"possibly_sensitive": False,
|
||||
"retweet_count": 13,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -1,226 +0,0 @@
|
|||
from unittest.mock import Mock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils.lorem_ipsum import words
|
||||
|
||||
from newsreader.accounts.tests.factories import UserFactory
|
||||
from newsreader.news.collection.exceptions import (
|
||||
StreamDeniedException,
|
||||
StreamException,
|
||||
StreamNotFoundException,
|
||||
StreamParseException,
|
||||
StreamTimeOutException,
|
||||
StreamTooManyException,
|
||||
)
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.twitter import TwitterClient
|
||||
|
||||
from .mocks import simple_mock
|
||||
|
||||
|
||||
class TwitterClientTestCase(TestCase):
|
||||
def setUp(self):
|
||||
patched_read = patch("newsreader.news.collection.twitter.TwitterStream.read")
|
||||
self.mocked_read = patched_read.start()
|
||||
|
||||
def tearDown(self):
|
||||
patch.stopall()
|
||||
|
||||
def test_simple(self):
|
||||
timeline = TwitterTimelineFactory()
|
||||
mock_stream = Mock(rule=timeline)
|
||||
|
||||
self.mocked_read.return_value = (simple_mock, mock_stream)
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertEquals(data, simple_mock)
|
||||
self.assertEquals(stream, mock_stream)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_stream_exception(self):
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
self.mocked_read.side_effect = StreamException(message="Stream exception")
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Stream exception")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_stream_not_found_exception(self):
|
||||
timeline = TwitterTimelineFactory.create()
|
||||
|
||||
self.mocked_read.side_effect = StreamNotFoundException(
|
||||
message="Stream not found"
|
||||
)
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Stream not found")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_stream_denied_exception(self):
|
||||
"""
|
||||
Twitter also returns these responses for accounts which have been shutdown.
|
||||
Therefore the error codes should be checked inside the response body.
|
||||
See https://stackoverflow.com/questions/8357568/do-twitter-access-token-expire
|
||||
"""
|
||||
user = UserFactory(
|
||||
twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
|
||||
)
|
||||
timeline = TwitterTimelineFactory(user=user)
|
||||
|
||||
self.mocked_read.side_effect = StreamDeniedException(message="Not authorized")
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Authorization required")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
user.refresh_from_db()
|
||||
timeline.refresh_from_db()
|
||||
|
||||
self.assertIsNotNone(user.twitter_oauth_token)
|
||||
self.assertIsNotNone(user.twitter_oauth_token_secret)
|
||||
|
||||
def test_client_catches_stream_timed_out_exception(self):
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
self.mocked_read.side_effect = StreamTimeOutException(
|
||||
message="Stream timed out"
|
||||
)
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Stream timed out")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_stream_too_many_exception(self):
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
self.mocked_read.side_effect = StreamTooManyException
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Too many requests")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_stream_parse_exception(self):
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
self.mocked_read.side_effect = StreamParseException(
|
||||
message="Stream could not be parsed"
|
||||
)
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Stream could not be parsed")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_long_exception_text(self):
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
self.mocked_read.side_effect = StreamParseException(message=words(1000))
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(len(stream.rule.error), 1024)
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
def test_client_catches_token_expired(self):
|
||||
user = UserFactory(
|
||||
twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
|
||||
)
|
||||
timeline = TwitterTimelineFactory(user=user)
|
||||
|
||||
response = Mock(json=lambda: {"errors": [{"code": 89}]})
|
||||
|
||||
self.mocked_read.side_effect = StreamDeniedException(
|
||||
message="Not authorized", response=response
|
||||
)
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "Authorization required")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
user.refresh_from_db()
|
||||
timeline.refresh_from_db()
|
||||
|
||||
self.assertIsNone(user.twitter_oauth_token)
|
||||
self.assertIsNone(user.twitter_oauth_token_secret)
|
||||
|
||||
def test_client_does_not_reset_token(self):
|
||||
"""
|
||||
The user's token and refresh token should not be reset when an generic
|
||||
exception is caught
|
||||
"""
|
||||
user = UserFactory(
|
||||
twitter_oauth_token=str(uuid4()), twitter_oauth_token_secret=str(uuid4())
|
||||
)
|
||||
timeline = TwitterTimelineFactory(user=user)
|
||||
|
||||
response = Mock(json=lambda: {"errors": [{"code": 100}]})
|
||||
|
||||
self.mocked_read.side_effect = StreamException(
|
||||
message="Generic message", response=response
|
||||
)
|
||||
|
||||
with TwitterClient([timeline]) as client:
|
||||
for data, stream in client:
|
||||
with self.subTest(data=data, stream=stream):
|
||||
self.assertIsNone(data)
|
||||
self.assertIsNone(stream)
|
||||
self.assertEquals(stream.rule.error, "")
|
||||
self.assertEquals(stream.rule.succeeded, False)
|
||||
|
||||
self.mocked_read.assert_called()
|
||||
|
||||
user.refresh_from_db()
|
||||
timeline.refresh_from_db()
|
||||
|
||||
self.assertIsNotNone(user.twitter_oauth_token)
|
||||
self.assertIsNotNone(user.twitter_oauth_token_secret)
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
# retrieved with:
|
||||
# curl -X GET -H "Authorization: Bearer <TOKEN>" "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys
|
||||
|
||||
simple_mock = [
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 20:32:22 +0000 2020",
|
||||
"display_text_range": [0, 111],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}],
|
||||
"symbols": [],
|
||||
"urls": [],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"favorite_count": 54,
|
||||
"favorited": False,
|
||||
"full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?",
|
||||
"geo": None,
|
||||
"id": 1307054882210435074,
|
||||
"id_str": "1307054882210435074",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"retweet_count": 9,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 18:50:11 +0000 2020",
|
||||
"display_text_range": [0, 271],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [211, 218], "text": "Twitch"}],
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
],
|
||||
"symbols": [],
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "twitch.tv/starcitizen",
|
||||
"expanded_url": "http://twitch.tv/starcitizen",
|
||||
"indices": [248, 271],
|
||||
"url": "https://t.co/2AdNovhpFW",
|
||||
}
|
||||
],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"extended_entities": {
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
]
|
||||
},
|
||||
"favorite_count": 90,
|
||||
"favorited": False,
|
||||
"full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9",
|
||||
"geo": None,
|
||||
"id": 1307029168941461504,
|
||||
"id_str": "1307029168941461504",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"possibly_sensitive": False,
|
||||
"retweet_count": 13,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
empty_mock = []
|
||||
|
|
@ -1,202 +0,0 @@
|
|||
from datetime import datetime
|
||||
from unittest.mock import Mock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
import pytz
|
||||
|
||||
from freezegun import freeze_time
|
||||
from ftfy import fix_text
|
||||
|
||||
from newsreader.news.collection.choices import RuleTypeChoices
|
||||
from newsreader.news.collection.exceptions import (
|
||||
StreamDeniedException,
|
||||
StreamForbiddenException,
|
||||
StreamNotFoundException,
|
||||
StreamTimeOutException,
|
||||
)
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.tests.twitter.collector.mocks import (
|
||||
empty_mock,
|
||||
simple_mock,
|
||||
)
|
||||
from newsreader.news.collection.twitter import TWITTER_URL, TwitterCollector
|
||||
from newsreader.news.collection.utils import truncate_text
|
||||
from newsreader.news.core.models import Post
|
||||
|
||||
|
||||
@freeze_time("2020-09-26 14:40:00")
|
||||
class TwitterCollectorTestCase(TestCase):
|
||||
def setUp(self):
|
||||
patched_get = patch("newsreader.news.collection.twitter.fetch")
|
||||
self.mocked_fetch = patched_get.start()
|
||||
|
||||
patched_parse = patch("newsreader.news.collection.twitter.TwitterStream.parse")
|
||||
self.mocked_parse = patched_parse.start()
|
||||
|
||||
def tearDown(self):
|
||||
patch.stopall()
|
||||
|
||||
def test_simple_batch(self):
|
||||
self.mocked_parse.return_value = simple_mock
|
||||
|
||||
timeline = TwitterTimelineFactory(
|
||||
user__twitter_oauth_token=str(uuid4()),
|
||||
user__twitter_oauth_token_secret=str(uuid4()),
|
||||
screen_name="RobertsSpaceInd",
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertCountEqual(
|
||||
Post.objects.values_list("remote_identifier", flat=True),
|
||||
("1307054882210435074", "1307029168941461504"),
|
||||
)
|
||||
|
||||
self.assertEquals(timeline.succeeded, True)
|
||||
self.assertEquals(timeline.last_run, timezone.now())
|
||||
self.assertIsNone(timeline.error)
|
||||
|
||||
post = Post.objects.get(
|
||||
remote_identifier="1307054882210435074",
|
||||
rule__type=RuleTypeChoices.twitter_timeline,
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 20, 32, 22))
|
||||
)
|
||||
|
||||
title = truncate_text(
|
||||
Post,
|
||||
"title",
|
||||
"It's a close match-up for #SCShipShowdown today! Which Aegis ship"
|
||||
" do you think will make it to the Semi-Finals?",
|
||||
)
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(post.title, title)
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1307054882210435074"
|
||||
)
|
||||
|
||||
post = Post.objects.get(
|
||||
remote_identifier="1307029168941461504",
|
||||
rule__type=RuleTypeChoices.twitter_timeline,
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
post.publication_date, pytz.utc.localize(datetime(2020, 9, 18, 18, 50, 11))
|
||||
)
|
||||
|
||||
body = fix_text(
|
||||
"We\u2019re welcoming members of our Builds, Publishes and Platform"
|
||||
" teams on Star Citizen Live to talk about the process involved in"
|
||||
" bringing everyone\u2019s work together and getting it out into your"
|
||||
" hands. Going live on #Twitch in 10 minutes."
|
||||
" \ud83c\udfa5\ud83d\udd34 \n\nTune in:"
|
||||
" https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9"
|
||||
)
|
||||
|
||||
title = truncate_text(Post, "title", body)
|
||||
|
||||
self.assertEquals(post.author, "RobertsSpaceInd")
|
||||
self.assertEquals(post.title, title)
|
||||
self.assertEquals(
|
||||
post.url, f"{TWITTER_URL}/RobertsSpaceInd/status/1307029168941461504"
|
||||
)
|
||||
|
||||
def test_empty_batch(self):
|
||||
self.mocked_parse.return_value = empty_mock
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
|
||||
self.assertEquals(timeline.succeeded, True)
|
||||
self.assertEquals(timeline.last_run, timezone.now())
|
||||
self.assertIsNone(timeline.error)
|
||||
|
||||
def test_not_found(self):
|
||||
self.mocked_fetch.side_effect = StreamNotFoundException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream not found")
|
||||
|
||||
def test_denied(self):
|
||||
self.mocked_fetch.side_effect = StreamDeniedException
|
||||
|
||||
timeline = TwitterTimelineFactory(
|
||||
user__twitter_oauth_token=str(uuid4()),
|
||||
user__twitter_oauth_token_secret=str(uuid4()),
|
||||
)
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream does not have sufficient permissions")
|
||||
|
||||
user = timeline.user
|
||||
|
||||
self.assertIsNotNone(user.twitter_oauth_token)
|
||||
self.assertIsNotNone(user.twitter_oauth_token_secret)
|
||||
|
||||
def test_forbidden(self):
|
||||
self.mocked_fetch.side_effect = StreamForbiddenException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream forbidden")
|
||||
|
||||
def test_timed_out(self):
|
||||
self.mocked_fetch.side_effect = StreamTimeOutException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream timed out")
|
||||
|
||||
def test_token_expired(self):
|
||||
response = Mock(json=lambda: {"errors": [{"code": 89}]})
|
||||
|
||||
self.mocked_fetch.side_effect = StreamDeniedException(response=response)
|
||||
|
||||
timeline = TwitterTimelineFactory(
|
||||
user__twitter_oauth_token=str(uuid4()),
|
||||
user__twitter_oauth_token_secret=str(uuid4()),
|
||||
)
|
||||
|
||||
collector = TwitterCollector()
|
||||
collector.collect(rules=[timeline])
|
||||
|
||||
self.assertEquals(Post.objects.count(), 0)
|
||||
self.assertEquals(timeline.succeeded, False)
|
||||
self.assertEquals(timeline.error, "Stream does not have sufficient permissions")
|
||||
|
||||
user = timeline.user
|
||||
|
||||
self.assertIsNone(user.twitter_oauth_token)
|
||||
self.assertIsNone(user.twitter_oauth_token_secret)
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
# retrieved with:
|
||||
# curl -X GET -H "Authorization: Bearer <TOKEN>" "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended" | python3 -m json.tool --sort-keys
|
||||
|
||||
simple_mock = [
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 20:32:22 +0000 2020",
|
||||
"display_text_range": [0, 111],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [26, 41], "text": "SCShipShowdown"}],
|
||||
"symbols": [],
|
||||
"urls": [],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"favorite_count": 54,
|
||||
"favorited": False,
|
||||
"full_text": "It's a close match-up for #SCShipShowdown today! Which Aegis ship do you think will make it to the Semi-Finals?",
|
||||
"geo": None,
|
||||
"id": 1307054882210435074,
|
||||
"id_str": "1307054882210435074",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"retweet_count": 9,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
{
|
||||
"contributors": None,
|
||||
"coordinates": None,
|
||||
"created_at": "Fri Sep 18 18:50:11 +0000 2020",
|
||||
"display_text_range": [0, 271],
|
||||
"entities": {
|
||||
"hashtags": [{"indices": [211, 218], "text": "Twitch"}],
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
],
|
||||
"symbols": [],
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "twitch.tv/starcitizen",
|
||||
"expanded_url": "http://twitch.tv/starcitizen",
|
||||
"indices": [248, 271],
|
||||
"url": "https://t.co/2AdNovhpFW",
|
||||
}
|
||||
],
|
||||
"user_mentions": [],
|
||||
},
|
||||
"extended_entities": {
|
||||
"media": [
|
||||
{
|
||||
"display_url": "pic.twitter.com/Cey5JpR1i9",
|
||||
"expanded_url": "https://twitter.com/RobertsSpaceInd/status/1307029168941461504/photo/1",
|
||||
"id": 1307028141697765376,
|
||||
"id_str": "1307028141697765376",
|
||||
"indices": [272, 295],
|
||||
"media_url": "http://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"media_url_https": "https://pbs.twimg.com/media/EiN_K4FVkAAGBcr.jpg",
|
||||
"sizes": {
|
||||
"large": {"h": 1090, "resize": "fit", "w": 1920},
|
||||
"medium": {"h": 681, "resize": "fit", "w": 1200},
|
||||
"small": {"h": 386, "resize": "fit", "w": 680},
|
||||
"thumb": {"h": 150, "resize": "crop", "w": 150},
|
||||
},
|
||||
"type": "photo",
|
||||
"url": "https://t.co/Cey5JpR1i9",
|
||||
}
|
||||
]
|
||||
},
|
||||
"favorite_count": 90,
|
||||
"favorited": False,
|
||||
"full_text": "We\u2019re welcoming members of our Builds, Publishes and Platform teams on Star Citizen Live to talk about the process involved in bringing everyone\u2019s work together and getting it out into your hands. Going live on #Twitch in 10 minutes. \ud83c\udfa5\ud83d\udd34 \n\nTune in: https://t.co/2AdNovhpFW https://t.co/Cey5JpR1i9",
|
||||
"geo": None,
|
||||
"id": 1307029168941461504,
|
||||
"id_str": "1307029168941461504",
|
||||
"in_reply_to_screen_name": None,
|
||||
"in_reply_to_status_id": None,
|
||||
"in_reply_to_status_id_str": None,
|
||||
"in_reply_to_user_id": None,
|
||||
"in_reply_to_user_id_str": None,
|
||||
"is_quote_status": False,
|
||||
"lang": "en",
|
||||
"place": None,
|
||||
"possibly_sensitive": False,
|
||||
"retweet_count": 13,
|
||||
"retweeted": False,
|
||||
"source": '<a href="https://mobile.twitter.com" rel="nofollow">Twitter Web App</a>',
|
||||
"truncated": False,
|
||||
"user": {
|
||||
"contributors_enabled": False,
|
||||
"created_at": "Wed Sep 05 00:58:11 +0000 2012",
|
||||
"default_profile": False,
|
||||
"default_profile_image": False,
|
||||
"description": "The official Twitter profile for #StarCitizen and Roberts Space Industries.",
|
||||
"entities": {
|
||||
"description": {"urls": []},
|
||||
"url": {
|
||||
"urls": [
|
||||
{
|
||||
"display_url": "robertsspaceindustries.com",
|
||||
"expanded_url": "http://www.robertsspaceindustries.com",
|
||||
"indices": [0, 23],
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
"favourites_count": 4831,
|
||||
"follow_request_sent": None,
|
||||
"followers_count": 106971,
|
||||
"following": None,
|
||||
"friends_count": 204,
|
||||
"geo_enabled": False,
|
||||
"has_extended_profile": False,
|
||||
"id": 803542770,
|
||||
"id_str": "803542770",
|
||||
"is_translation_enabled": False,
|
||||
"is_translator": False,
|
||||
"lang": None,
|
||||
"listed_count": 893,
|
||||
"location": "Roberts Space Industries",
|
||||
"name": "Star Citizen",
|
||||
"notifications": None,
|
||||
"profile_background_color": "131516",
|
||||
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif",
|
||||
"profile_background_tile": False,
|
||||
"profile_banner_url": "https://pbs.twimg.com/profile_banners/803542770/1596651186",
|
||||
"profile_image_url": "http://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/profile_images/963109950103814144/ysnj_Asy_normal.jpg",
|
||||
"profile_link_color": "0A5485",
|
||||
"profile_sidebar_border_color": "FFFFFF",
|
||||
"profile_sidebar_fill_color": "EFEFEF",
|
||||
"profile_text_color": "333333",
|
||||
"profile_use_background_image": True,
|
||||
"protected": False,
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"statuses_count": 6368,
|
||||
"time_zone": None,
|
||||
"translator_type": "none",
|
||||
"url": "https://t.co/iqO6apof3y",
|
||||
"utc_offset": None,
|
||||
"verified": True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
from json import JSONDecodeError
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from newsreader.news.collection.exceptions import (
|
||||
StreamDeniedException,
|
||||
StreamException,
|
||||
StreamForbiddenException,
|
||||
StreamNotFoundException,
|
||||
StreamParseException,
|
||||
StreamTimeOutException,
|
||||
)
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.tests.twitter.stream.mocks import simple_mock
|
||||
from newsreader.news.collection.twitter import TwitterStream
|
||||
|
||||
|
||||
class TwitterStreamTestCase(TestCase):
|
||||
def setUp(self):
|
||||
self.patched_fetch = patch("newsreader.news.collection.twitter.fetch")
|
||||
self.mocked_fetch = self.patched_fetch.start()
|
||||
|
||||
def tearDown(self):
|
||||
patch.stopall()
|
||||
|
||||
def test_simple_stream(self):
|
||||
self.mocked_fetch.return_value.json.return_value = simple_mock
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
data, stream = stream.read()
|
||||
|
||||
self.assertEquals(data, simple_mock)
|
||||
self.assertEquals(stream, stream)
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_denied_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamDeniedException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamDeniedException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_not_found_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamNotFoundException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamNotFoundException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_time_out_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamTimeOutException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamTimeOutException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_forbidden_exception(self):
|
||||
self.mocked_fetch.side_effect = StreamForbiddenException
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamForbiddenException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
||||
def test_stream_raises_parse_exception(self):
|
||||
self.mocked_fetch.return_value.json.side_effect = JSONDecodeError(
|
||||
"No json found", "{}", 5
|
||||
)
|
||||
|
||||
timeline = TwitterTimelineFactory()
|
||||
stream = TwitterStream(timeline)
|
||||
|
||||
with self.assertRaises(StreamParseException):
|
||||
stream.read()
|
||||
|
||||
self.mocked_fetch.assert_called()
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
from json import JSONDecodeError
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from newsreader.accounts.tests.factories import UserFactory
|
||||
from newsreader.news.collection.exceptions import StreamException
|
||||
from newsreader.news.collection.twitter import TwitterTimeLineScheduler
|
||||
|
||||
|
||||
class TwitterTimeLineSchedulerTestCase(TestCase):
|
||||
def setUp(self):
|
||||
patched_fetch = patch("newsreader.news.collection.twitter.fetch")
|
||||
self.mocked_fetch = patched_fetch.start()
|
||||
|
||||
def test_simple(self):
|
||||
user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar")
|
||||
|
||||
self.mocked_fetch.return_value.json.return_value = {
|
||||
"rate_limit_context": {"application": "dummykey"},
|
||||
"resources": {
|
||||
"statuses": {
|
||||
"/statuses/user_timeline": {
|
||||
"limit": 1500,
|
||||
"remaining": 1500,
|
||||
"reset": 1601141386,
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
scheduler = TwitterTimeLineScheduler(user)
|
||||
|
||||
self.assertEquals(scheduler.get_current_ratelimit(), 1500)
|
||||
|
||||
def test_stream_exception(self):
|
||||
user = UserFactory(twitter_oauth_token=None, twitter_oauth_token_secret=None)
|
||||
|
||||
self.mocked_fetch.side_effect = StreamException
|
||||
|
||||
scheduler = TwitterTimeLineScheduler(user)
|
||||
|
||||
self.assertEquals(scheduler.get_current_ratelimit(), None)
|
||||
|
||||
def test_json_decode_error(self):
|
||||
user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar")
|
||||
|
||||
self.mocked_fetch.return_value.json.side_effect = JSONDecodeError(
|
||||
"foo", "bar", 10
|
||||
)
|
||||
|
||||
scheduler = TwitterTimeLineScheduler(user)
|
||||
|
||||
self.assertEquals(scheduler.get_current_ratelimit(), None)
|
||||
|
||||
def test_unexpected_contents(self):
|
||||
user = UserFactory(twitter_oauth_token="foo", twitter_oauth_token_secret="bar")
|
||||
|
||||
self.mocked_fetch.return_value.json.return_value = {"foo": "bar"}
|
||||
|
||||
scheduler = TwitterTimeLineScheduler(user)
|
||||
|
||||
self.assertEquals(scheduler.get_current_ratelimit(), None)
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
from django.test import TestCase
|
||||
from django.urls import reverse
|
||||
|
||||
import pytz
|
||||
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
|
||||
from newsreader.news.collection.choices import RuleTypeChoices
|
||||
from newsreader.news.collection.models import CollectionRule
|
||||
from newsreader.news.collection.tests.factories import TwitterTimelineFactory
|
||||
from newsreader.news.collection.tests.views.base import CollectionRuleViewTestCase
|
||||
from newsreader.news.collection.twitter import TWITTER_API_URL
|
||||
from newsreader.news.core.tests.factories import CategoryFactory
|
||||
|
||||
|
||||
class TwitterTimelineCreateViewTestCase(CollectionRuleViewTestCase, TestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
self.form_data = {
|
||||
"name": "new rule",
|
||||
"screen_name": "RobertsSpaceInd",
|
||||
"category": str(self.category.pk),
|
||||
}
|
||||
|
||||
self.url = reverse("news:collection:twitter-timeline-create")
|
||||
|
||||
def test_creation(self):
|
||||
response = self.client.post(self.url, self.form_data)
|
||||
|
||||
self.assertEquals(response.status_code, 302)
|
||||
|
||||
rule = CollectionRule.objects.get(name="new rule")
|
||||
|
||||
self.assertEquals(rule.type, RuleTypeChoices.twitter_timeline)
|
||||
self.assertEquals(
|
||||
rule.url,
|
||||
f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name=RobertsSpaceInd&tweet_mode=extended",
|
||||
)
|
||||
self.assertEquals(rule.timezone, str(pytz.utc))
|
||||
self.assertEquals(rule.favicon, None)
|
||||
self.assertEquals(rule.category.pk, self.category.pk)
|
||||
self.assertEquals(rule.user.pk, self.user.pk)
|
||||
|
||||
self.assertTrue(
|
||||
PeriodicTask.objects.get(
|
||||
name=f"{self.user.email}-timeline",
|
||||
task="TwitterTimelineTask",
|
||||
enabled=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TwitterTimelineUpdateViewTestCase(CollectionRuleViewTestCase, TestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
self.rule = TwitterTimelineFactory(
|
||||
name="Star citizen",
|
||||
screen_name="RobertsSpaceInd",
|
||||
user=self.user,
|
||||
category=self.category,
|
||||
type=RuleTypeChoices.twitter_timeline,
|
||||
)
|
||||
self.url = reverse(
|
||||
"news:collection:twitter-timeline-update", kwargs={"pk": self.rule.pk}
|
||||
)
|
||||
|
||||
self.form_data = {
|
||||
"name": self.rule.name,
|
||||
"screen_name": self.rule.screen_name,
|
||||
"category": str(self.category.pk),
|
||||
"timezone": pytz.utc,
|
||||
}
|
||||
|
||||
def test_name_change(self):
|
||||
self.form_data.update(name="Star citizen Twitter")
|
||||
|
||||
response = self.client.post(self.url, self.form_data)
|
||||
self.assertEquals(response.status_code, 302)
|
||||
|
||||
self.rule.refresh_from_db()
|
||||
|
||||
self.assertEquals(self.rule.name, "Star citizen Twitter")
|
||||
|
||||
def test_category_change(self):
|
||||
new_category = CategoryFactory(user=self.user)
|
||||
|
||||
self.form_data.update(category=new_category.pk)
|
||||
|
||||
response = self.client.post(self.url, self.form_data)
|
||||
self.assertEquals(response.status_code, 302)
|
||||
|
||||
self.rule.refresh_from_db()
|
||||
|
||||
self.assertEquals(self.rule.category.pk, new_category.pk)
|
||||
|
||||
def test_twitter_timelines_only(self):
|
||||
rule = TwitterTimelineFactory(
|
||||
name="Fake twitter",
|
||||
user=self.user,
|
||||
category=self.category,
|
||||
type=RuleTypeChoices.feed,
|
||||
url="https://twitter.com/RobertsSpaceInd",
|
||||
)
|
||||
url = reverse("news:collection:twitter-timeline-update", kwargs={"pk": rule.pk})
|
||||
|
||||
response = self.client.get(url)
|
||||
|
||||
self.assertEquals(response.status_code, 404)
|
||||
|
||||
def test_screen_name_change(self):
|
||||
self.form_data.update(screen_name="CyberpunkGame")
|
||||
|
||||
response = self.client.post(self.url, self.form_data)
|
||||
|
||||
self.assertEquals(response.status_code, 302)
|
||||
|
||||
self.rule.refresh_from_db()
|
||||
|
||||
self.assertEquals(self.rule.type, RuleTypeChoices.twitter_timeline)
|
||||
self.assertEquals(
|
||||
self.rule.url,
|
||||
f"{TWITTER_API_URL}/statuses/user_timeline.json?screen_name=CyberpunkGame&tweet_mode=extended",
|
||||
)
|
||||
self.assertEquals(self.rule.timezone, str(pytz.utc))
|
||||
self.assertEquals(self.rule.favicon, None)
|
||||
self.assertEquals(self.rule.category.pk, self.category.pk)
|
||||
self.assertEquals(self.rule.user.pk, self.user.pk)
|
||||
|
|
@ -1,348 +0,0 @@
|
|||
import logging
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from json import JSONDecodeError
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.mail import send_mail
|
||||
from django.utils import timezone
|
||||
from django.utils.html import format_html, urlize
|
||||
from django.utils.translation import ugettext as _
|
||||
|
||||
import pytz
|
||||
|
||||
from ftfy import fix_text
|
||||
from requests_oauthlib import OAuth1 as OAuth
|
||||
|
||||
from newsreader.news.collection.base import (
|
||||
PostBuilder,
|
||||
PostClient,
|
||||
PostCollector,
|
||||
PostStream,
|
||||
Scheduler,
|
||||
)
|
||||
from newsreader.news.collection.choices import RuleTypeChoices, TwitterPostTypeChoices
|
||||
from newsreader.news.collection.exceptions import (
|
||||
BuilderDuplicateException,
|
||||
BuilderException,
|
||||
BuilderMissingDataException,
|
||||
BuilderParseException,
|
||||
StreamException,
|
||||
StreamNotFoundException,
|
||||
StreamParseException,
|
||||
StreamTimeOutException,
|
||||
StreamTooManyException,
|
||||
)
|
||||
from newsreader.news.collection.utils import fetch, truncate_text
|
||||
from newsreader.news.core.models import Post
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TWITTER_URL = "https://twitter.com"
|
||||
TWITTER_API_URL = "https://api.twitter.com/1.1"
|
||||
TWITTER_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token"
|
||||
TWITTER_AUTH_URL = "https://api.twitter.com/oauth/authorize"
|
||||
TWITTER_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token"
|
||||
TWITTER_REVOKE_URL = f"{TWITTER_API_URL}/oauth/invalidate_token"
|
||||
|
||||
|
||||
class TwitterBuilder(PostBuilder):
|
||||
rule_type = RuleTypeChoices.twitter_timeline
|
||||
|
||||
def build(self):
|
||||
results = {}
|
||||
|
||||
for post in self.payload:
|
||||
try:
|
||||
post = self.build_post(post)
|
||||
except BuilderDuplicateException:
|
||||
logger.warning("Skipping duplicate post")
|
||||
continue
|
||||
except BuilderException:
|
||||
logger.exception("Failed building post")
|
||||
continue
|
||||
|
||||
identifier = post.remote_identifier
|
||||
results[identifier] = post
|
||||
|
||||
self.instances = results.values()
|
||||
|
||||
def build_post(self, data):
|
||||
remote_identifier = data.get("id_str", "")
|
||||
rule = self.stream.rule
|
||||
|
||||
if remote_identifier in self.existing_posts:
|
||||
raise BuilderDuplicateException(payload=data)
|
||||
|
||||
try:
|
||||
body = urlize(data["full_text"], nofollow=True)
|
||||
title = truncate_text(
|
||||
Post, "title", self.sanitize_fragment(data["full_text"])
|
||||
)
|
||||
|
||||
publication_date = pytz.utc.localize(
|
||||
datetime.strptime(data["created_at"], "%a %b %d %H:%M:%S +0000 %Y")
|
||||
)
|
||||
except KeyError as e:
|
||||
raise BuilderMissingDataException(payload=data) from e
|
||||
except (OverflowError, OSError) as e:
|
||||
raise BuilderParseException(payload=data) from e
|
||||
|
||||
url = f"{TWITTER_URL}/{rule.screen_name}/status/{remote_identifier}"
|
||||
|
||||
if "extended_entities" in data:
|
||||
try:
|
||||
media_entities = self.get_media_entities(data)
|
||||
body += media_entities
|
||||
except KeyError as e:
|
||||
raise BuilderMissingDataException(
|
||||
message="Failed parsing data for media entities", payload=data
|
||||
) from e
|
||||
|
||||
try:
|
||||
if "retweeted_status" in data:
|
||||
original_post = data["retweeted_status"]
|
||||
original_tweet = urlize(original_post["full_text"], nofollow=True)
|
||||
body = f"{body} <br><div>Original tweet: {original_tweet}</div>"
|
||||
if "quoted_status" in data:
|
||||
original_post = data["quoted_status"]
|
||||
original_tweet = urlize(original_post["full_text"], nofollow=True)
|
||||
body = f"{body} <br><div>Quoted tweet: {original_tweet}</div>"
|
||||
except KeyError as e:
|
||||
raise BuilderMissingDataException(
|
||||
message="Failed parsing data for original tweet", payload=data
|
||||
) from e
|
||||
|
||||
body = self.sanitize_fragment(body)
|
||||
|
||||
return Post(
|
||||
**{
|
||||
"remote_identifier": remote_identifier,
|
||||
"title": fix_text(title),
|
||||
"body": fix_text(body),
|
||||
"author": rule.screen_name,
|
||||
"publication_date": publication_date,
|
||||
"url": url,
|
||||
"rule": rule,
|
||||
}
|
||||
)
|
||||
|
||||
def get_media_entities(self, data):
|
||||
media_entities = data["extended_entities"]["media"]
|
||||
formatted_entities = ""
|
||||
|
||||
for media_entity in media_entities:
|
||||
media_type = media_entity["type"]
|
||||
media_url = media_entity["media_url_https"]
|
||||
title = media_entity["id_str"]
|
||||
|
||||
if media_type == TwitterPostTypeChoices.photo:
|
||||
html_fragment = format_html(
|
||||
"""<br /><div><img alt="{title}" src="{media_url}" loading="lazy" /></div>""",
|
||||
title=title,
|
||||
media_url=media_url,
|
||||
)
|
||||
|
||||
formatted_entities += html_fragment
|
||||
|
||||
elif media_type in (
|
||||
TwitterPostTypeChoices.video,
|
||||
TwitterPostTypeChoices.animated_gif,
|
||||
):
|
||||
meta_data = media_entity["video_info"]
|
||||
|
||||
videos = sorted(
|
||||
[video for video in meta_data["variants"]],
|
||||
reverse=True,
|
||||
key=lambda video: video.get("bitrate", 0),
|
||||
)
|
||||
|
||||
if not videos:
|
||||
continue
|
||||
|
||||
video = videos[0]
|
||||
content_type = video["content_type"]
|
||||
url = video["url"]
|
||||
|
||||
html_fragment = format_html(
|
||||
"""<br /><div><video controls muted><source src="{url}" type="{content_type}" /></video></div> """,
|
||||
url=url,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
formatted_entities += html_fragment
|
||||
|
||||
return formatted_entities
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.stream.rule.pk}: TwitterBuilder"
|
||||
|
||||
|
||||
class TwitterStream(PostStream):
|
||||
rule_type = RuleTypeChoices.twitter_timeline
|
||||
|
||||
def read(self):
|
||||
oauth = OAuth(
|
||||
settings.TWITTER_CONSUMER_ID,
|
||||
client_secret=settings.TWITTER_CONSUMER_SECRET,
|
||||
resource_owner_key=self.rule.user.twitter_oauth_token,
|
||||
resource_owner_secret=self.rule.user.twitter_oauth_token_secret,
|
||||
)
|
||||
|
||||
response = fetch(self.rule.url, auth=oauth)
|
||||
|
||||
return self.parse(response), self
|
||||
|
||||
def parse(self, response):
|
||||
try:
|
||||
return response.json()
|
||||
except JSONDecodeError as e:
|
||||
raise StreamParseException(
|
||||
response=response, message="Failed parsing json"
|
||||
) from e
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.rule.pk}: TwitterStream"
|
||||
|
||||
|
||||
class TwitterClient(PostClient):
|
||||
stream = TwitterStream
|
||||
|
||||
def __enter__(self):
|
||||
streams = [self.stream(timeline) for timeline in self.rules]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures = {executor.submit(stream.read): stream for stream in streams}
|
||||
|
||||
for future in as_completed(futures):
|
||||
stream = futures[future]
|
||||
|
||||
try:
|
||||
payload = future.result()
|
||||
|
||||
stream.rule.error = None
|
||||
stream.rule.succeeded = True
|
||||
|
||||
yield payload
|
||||
except StreamTooManyException as e:
|
||||
logger.exception("Ratelimit hit, aborting twitter calls")
|
||||
|
||||
self.set_rule_error(stream.rule, e)
|
||||
|
||||
break
|
||||
except (StreamNotFoundException, StreamTimeOutException) as e:
|
||||
logger.warning(f"Request failed for {stream.rule.screen_name}")
|
||||
|
||||
self.set_rule_error(stream.rule, e)
|
||||
|
||||
continue
|
||||
except StreamException as e:
|
||||
logger.exception(f"Request failed for {stream.rule.screen_name}")
|
||||
|
||||
self.set_rule_error(stream.rule, e)
|
||||
|
||||
if not e.response:
|
||||
continue
|
||||
|
||||
try:
|
||||
response_data = e.response.json()
|
||||
except JSONDecodeError:
|
||||
logger.exception("Could not parse json for request")
|
||||
continue
|
||||
|
||||
if "errors" in response_data:
|
||||
errors = response_data["errors"]
|
||||
token_expired = any(error["code"] == 89 for error in errors)
|
||||
|
||||
if token_expired:
|
||||
try:
|
||||
import sentry_sdk
|
||||
|
||||
with sentry_sdk.push_scope() as scope:
|
||||
scope.set_extra("content", response_data)
|
||||
sentry_sdk.capture_message(
|
||||
"Twitter authentication credentials reset"
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
stream.rule.user.twitter_oauth_token = None
|
||||
stream.rule.user.twitter_oauth_token_secret = None
|
||||
stream.rule.user.save()
|
||||
|
||||
message = _(
|
||||
"Your Twitter account credentials have expired. Re-authenticate in"
|
||||
" the settings page to keep retrieving Twitter specific information"
|
||||
" from your account."
|
||||
)
|
||||
|
||||
send_mail(
|
||||
"Twitter account needs re-authentication",
|
||||
message,
|
||||
None,
|
||||
[stream.rule.user.email],
|
||||
)
|
||||
|
||||
continue
|
||||
finally:
|
||||
stream.rule.last_run = timezone.now()
|
||||
stream.rule.save()
|
||||
|
||||
|
||||
class TwitterCollector(PostCollector):
|
||||
builder = TwitterBuilder
|
||||
client = TwitterClient
|
||||
|
||||
|
||||
# see https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
|
||||
class TwitterTimeLineScheduler(Scheduler):
|
||||
def __init__(self, user, timelines=[]):
|
||||
self.user = user
|
||||
|
||||
if not timelines:
|
||||
self.timelines = (
|
||||
user.rules.enabled()
|
||||
.filter(type=RuleTypeChoices.twitter_timeline)
|
||||
.order_by("last_run")[:200]
|
||||
)
|
||||
else:
|
||||
self.timelines = timelines
|
||||
|
||||
def get_scheduled_rules(self):
|
||||
max_amount = self.get_current_ratelimit()
|
||||
return self.timelines[:max_amount] if max_amount else []
|
||||
|
||||
def get_current_ratelimit(self):
|
||||
endpoint = "application/rate_limit_status.json?resources=statuses"
|
||||
|
||||
if (
|
||||
not self.user.twitter_oauth_token
|
||||
or not self.user.twitter_oauth_token_secret
|
||||
):
|
||||
return
|
||||
|
||||
oauth = OAuth(
|
||||
settings.TWITTER_CONSUMER_ID,
|
||||
client_secret=settings.TWITTER_CONSUMER_SECRET,
|
||||
resource_owner_key=self.user.twitter_oauth_token,
|
||||
resource_owner_secret=self.user.twitter_oauth_token_secret,
|
||||
)
|
||||
|
||||
try:
|
||||
response = fetch(f"{TWITTER_API_URL}/{endpoint}", auth=oauth)
|
||||
except StreamException:
|
||||
logger.exception(f"Unable to retrieve current ratelimit for {self.user.pk}")
|
||||
return
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except JSONDecodeError:
|
||||
logger.exception(f"Unable to parse ratelimit request for {self.user.pk}")
|
||||
return
|
||||
|
||||
try:
|
||||
return payload["resources"]["statuses"]["/statuses/user_timeline"]["limit"]
|
||||
except KeyError:
|
||||
return
|
||||
|
|
@ -16,8 +16,6 @@ from newsreader.news.collection.views import (
|
|||
OPMLImportView,
|
||||
SubRedditCreateView,
|
||||
SubRedditUpdateView,
|
||||
TwitterTimelineCreateView,
|
||||
TwitterTimelineUpdateView,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -62,15 +60,4 @@ urlpatterns = [
|
|||
login_required(SubRedditUpdateView.as_view()),
|
||||
name="subreddit-update",
|
||||
),
|
||||
# Twitter
|
||||
path(
|
||||
"twitter/timelines/create/",
|
||||
login_required(TwitterTimelineCreateView.as_view()),
|
||||
name="twitter-timeline-create",
|
||||
),
|
||||
path(
|
||||
"twitter/timelines/<int:pk>/",
|
||||
login_required(TwitterTimelineUpdateView.as_view()),
|
||||
name="twitter-timeline-update",
|
||||
),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -13,10 +13,6 @@ from newsreader.news.collection.views.rules import (
|
|||
CollectionRuleBulkEnableView,
|
||||
CollectionRuleListView,
|
||||
)
|
||||
from newsreader.news.collection.views.twitter import (
|
||||
TwitterTimelineCreateView,
|
||||
TwitterTimelineUpdateView,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -29,6 +25,4 @@ __all__ = [
|
|||
"CollectionRuleBulkDisableView",
|
||||
"CollectionRuleBulkEnableView",
|
||||
"CollectionRuleListView",
|
||||
"TwitterTimelineCreateView",
|
||||
"TwitterTimelineUpdateView",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
from django.views.generic.edit import CreateView, UpdateView
|
||||
|
||||
from django_celery_beat.models import IntervalSchedule
|
||||
|
||||
from newsreader.news.collection.choices import RuleTypeChoices
|
||||
from newsreader.news.collection.forms import TwitterTimelineForm
|
||||
from newsreader.news.collection.views.base import (
|
||||
CollectionRuleDetailMixin,
|
||||
CollectionRuleViewMixin,
|
||||
TaskCreationMixin,
|
||||
)
|
||||
|
||||
|
||||
class TwitterTimelineCreateView(
|
||||
CollectionRuleViewMixin, CollectionRuleDetailMixin, TaskCreationMixin, CreateView
|
||||
):
|
||||
form_class = TwitterTimelineForm
|
||||
template_name = "news/collection/views/twitter/timeline-create.html"
|
||||
task_interval = (10, IntervalSchedule.MINUTES)
|
||||
task_name = "timeline"
|
||||
task_type = "TwitterTimelineTask"
|
||||
|
||||
|
||||
class TwitterTimelineUpdateView(
|
||||
CollectionRuleViewMixin, CollectionRuleDetailMixin, UpdateView
|
||||
):
|
||||
form_class = TwitterTimelineForm
|
||||
template_name = "news/collection/views/twitter/timeline-update.html"
|
||||
context_object_name = "timeline"
|
||||
|
||||
def get_queryset(self):
|
||||
queryset = super().get_queryset()
|
||||
return queryset.filter(type=RuleTypeChoices.twitter_timeline)
|
||||
|
|
@ -22,9 +22,6 @@ class NewsView(TemplateView):
|
|||
"subredditUrl": reverse_lazy(
|
||||
"news:collection:subreddit-update", args=(0,)
|
||||
),
|
||||
"timelineUrl": reverse_lazy(
|
||||
"news:collection:twitter-timeline-update", args=(0,)
|
||||
),
|
||||
"categoriesUrl": reverse_lazy("news:core:category-update", args=(0,)),
|
||||
"timezone": settings.TIME_ZONE,
|
||||
"autoMarking": self.request.user.auto_mark_read,
|
||||
|
|
|
|||
|
|
@ -38,15 +38,6 @@
|
|||
}
|
||||
}
|
||||
|
||||
&--twitter {
|
||||
color: $white !important;
|
||||
background-color: $twitter-blue;
|
||||
|
||||
&:hover {
|
||||
background-color: lighten($twitter-blue, 5%);
|
||||
}
|
||||
}
|
||||
|
||||
&--disabled {
|
||||
color: var(--font-color) !important;
|
||||
background-color: $gray !important;
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ $black: rgba(0, 0, 0, 1);
|
|||
$dark: rgba(0, 0, 0, 0.4);
|
||||
|
||||
$reddit-orange: rgba(255, 69, 0, 1);
|
||||
$twitter-blue: rgba(29, 155, 240, 1);
|
||||
|
||||
$transparant-red: transparentize($red, 0.8);
|
||||
$transparant-blue: transparentize($blue, 0.8);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue