sonny/newsreader
1
0
Fork 0
Code Issues 13 Pull requests 2 Projects Releases 58 Packages Wiki Activity Actions

0.3.6

Browse source 
- Update deploy job
- Add user manageable reddit filters
This commit is contained in:
Sonny Bakker 2020-12-19 21:03:29 +01:00
parent 116b6d1308
commit 04af0f9c5d
17 changed files with 1680 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

6
gitlab-ci/deploy.yml
View file

@ -8,15 +8,13 @@ deploy:
- if: $CI_COMMIT_TAG
before_script:
- pip install ansible --quiet
- git clone https://git.fudiggity.nl/sonny/ansible-playbooks.git deployment --branch master
- git clone https://git.fudiggity.nl/ansible/newsreader.git deployment --branch master
- mkdir /root/.ssh && echo "$DEPLOY_HOST_KEY" > /root/.ssh/known_hosts
- echo "$DEPLOY_KEY" > deployment/deploy_key && chmod 0600 deployment/deploy_key
- echo "$VAULT_PASSWORD" > deployment/vault
script:
- >
ansible-playbook deployment/playbook.yml
--inventory deployment/apps.yml
--limit newsreader
--user ansible
--inventory deployment/inventory.yml
--private-key deployment/deploy_key
--vault-password-file deployment/vault

2
pyproject.toml
View file

@ -40,5 +40,5 @@ tblib = "1.6.0"
coverage = "^5.1"
[build-system]
requires = ["poetry>=0.12"]
requires = ["poetry>=1.0.10"]
build-backend = "poetry.masonry.api"

49
src/newsreader/accounts/migrations/0014_auto_20201218_2216.py Normal file
View file

@ -0,0 +1,49 @@
# Generated by Django 3.0.7 on 2020-12-18 21:16
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("accounts", "0013_user_auto_mark_read")]
operations = [
migrations.AddField(
model_name="user",
name="reddit_allow_nfsw",
field=models.BooleanField(default=False, verbose_name="Allow NSFW posts"),
),
migrations.AddField(
model_name="user",
name="reddit_allow_spoiler",
field=models.BooleanField(default=False, verbose_name="Allow spoilers"),
),
migrations.AddField(
model_name="user",
name="reddit_allow_viewed",
field=models.BooleanField(
default=True, verbose_name="Allow already seen posts"
),
),
migrations.AddField(
model_name="user",
name="reddit_comments_min",
field=models.PositiveIntegerField(
default=0, verbose_name="Minimum amount of comments"
),
),
migrations.AddField(
model_name="user",
name="reddit_downvotes_max",
field=models.PositiveIntegerField(
default=0, verbose_name="Maximum amount of downvotes"
),
),
migrations.AddField(
model_name="user",
name="reddit_upvotes_min",
field=models.PositiveIntegerField(
default=0, verbose_name="Minimum amount of upvotes"
),
),
]

17
src/newsreader/accounts/migrations/0015_auto_20201219_1330.py Normal file
View file

@ -0,0 +1,17 @@
# Generated by Django 3.0.7 on 2020-12-19 12:30
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [("accounts", "0014_auto_20201218_2216")]
operations = [
migrations.RemoveField(model_name="user", name="reddit_allow_nfsw"),
migrations.RemoveField(model_name="user", name="reddit_allow_spoiler"),
migrations.RemoveField(model_name="user", name="reddit_allow_viewed"),
migrations.RemoveField(model_name="user", name="reddit_comments_min"),
migrations.RemoveField(model_name="user", name="reddit_downvotes_max"),
migrations.RemoveField(model_name="user", name="reddit_upvotes_min"),
]

2
src/newsreader/accounts/models.py
View file

@ -39,9 +39,11 @@ class UserManager(DjangoUserManager):
class User(AbstractUser):
email = models.EmailField(_("email address"), unique=True)
# reddit settings
reddit_refresh_token = models.CharField(max_length=255, blank=True, null=True)
reddit_access_token = models.CharField(max_length=255, blank=True, null=True)
# twitter settings
twitter_oauth_token = models.CharField(max_length=255, blank=True, null=True)
twitter_oauth_token_secret = models.CharField(max_length=255, blank=True, null=True)

1
src/newsreader/news/collection/exceptions/__init__.py
View file

@ -3,6 +3,7 @@ from newsreader.news.collection.exceptions.builder import (
BuilderException,
BuilderMissingDataException,
BuilderParseException,
BuilderSkippedException,
)
from newsreader.news.collection.exceptions.stream import (
StreamConnectionException,

4
src/newsreader/news/collection/exceptions/builder.py
View file

@ -19,3 +19,7 @@ class BuilderDuplicateException(BuilderException):
class BuilderParseException(BuilderException):
message = "Failed to parse payload"
class BuilderSkippedException(BuilderException):
message = "Payload does not meet filter criteria"

13
src/newsreader/news/collection/forms/reddit.py
View file

@ -46,4 +46,15 @@ class SubRedditForm(CollectionRuleForm):
class Meta:
model = CollectionRule
fields = ("name", "url", "favicon", "category")
fields = (
"name",
"url",
"favicon",
"category",
"reddit_allow_nfsw",
"reddit_allow_spoiler",
"reddit_allow_viewed",
"reddit_upvotes_min",
"reddit_downvotes_max",
"reddit_comments_min",
)

49
src/newsreader/news/collection/migrations/0012_auto_20201219_1331.py Normal file
View file

@ -0,0 +1,49 @@
# Generated by Django 3.0.7 on 2020-12-19 12:31
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("collection", "0011_auto_20200913_2157")]
operations = [
migrations.AddField(
model_name="collectionrule",
name="reddit_allow_nfsw",
field=models.BooleanField(default=False, verbose_name="Allow NSFW posts"),
),
migrations.AddField(
model_name="collectionrule",
name="reddit_allow_spoiler",
field=models.BooleanField(default=False, verbose_name="Allow spoilers"),
),
migrations.AddField(
model_name="collectionrule",
name="reddit_allow_viewed",
field=models.BooleanField(
default=True, verbose_name="Allow already seen posts"
),
),
migrations.AddField(
model_name="collectionrule",
name="reddit_comments_min",
field=models.PositiveIntegerField(
default=0, verbose_name="Minimum amount of comments"
),
),
migrations.AddField(
model_name="collectionrule",
name="reddit_downvotes_max",
field=models.PositiveIntegerField(
default=0, verbose_name="Maximum amount of downvotes"
),
),
migrations.AddField(
model_name="collectionrule",
name="reddit_upvotes_min",
field=models.PositiveIntegerField(
default=0, verbose_name="Minimum amount of upvotes"
),
),
]

18
src/newsreader/news/collection/migrations/0013_auto_20201219_1345.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.0.7 on 2020-12-19 12:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("collection", "0012_auto_20201219_1331")]
operations = [
migrations.AlterField(
model_name="collectionrule",
name="reddit_downvotes_max",
field=models.PositiveIntegerField(
blank=True, null=True, verbose_name="Maximum amount of downvotes"
),
)
]

18
src/newsreader/news/collection/migrations/0014_auto_20201219_1346.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.0.7 on 2020-12-19 12:46
from django.db import migrations
def reset_default_downvotes(apps, schema_editor):
CollectionRule = apps.get_model("collection", "CollectionRule")
for rule in CollectionRule.objects.all():
rule.reddit_downvotes_max = None
rule.save()
class Migration(migrations.Migration):
dependencies = [("collection", "0013_auto_20201219_1345")]
operations = [migrations.RunPython(reset_default_downvotes)]

16
src/newsreader/news/collection/models.py
View file

@ -56,6 +56,22 @@ class CollectionRule(TimeStampedModel):
on_delete=models.CASCADE,
)
# Reddit
reddit_allow_nfsw = models.BooleanField(_("Allow NSFW posts"), default=False)
reddit_allow_spoiler = models.BooleanField(_("Allow spoilers"), default=False)
reddit_allow_viewed = models.BooleanField(
_("Allow already seen posts"), default=True
)
reddit_upvotes_min = models.PositiveIntegerField(
_("Minimum amount of upvotes"), default=0
)
reddit_downvotes_max = models.PositiveIntegerField(
_("Maximum amount of downvotes"), blank=True, null=True
)
reddit_comments_min = models.PositiveIntegerField(
_("Minimum amount of comments"), default=0
)
# Twitter
screen_name = models.CharField(max_length=255, blank=True, null=True)

28
src/newsreader/news/collection/reddit.py
View file

@ -32,6 +32,7 @@ from newsreader.news.collection.exceptions import (
BuilderException,
BuilderMissingDataException,
BuilderParseException,
BuilderSkippedException,
StreamDeniedException,
StreamException,
StreamParseException,
@ -165,13 +166,40 @@ class RedditBuilder(PostBuilder):
try:
title = entry_data["title"]
author = entry_data["author"]
post_url_fragment = entry_data["permalink"]
direct_url = entry_data["url"]
is_text = entry_data["is_self"]
is_video = entry_data["is_video"]
is_nsfw = entry_data["over_18"]
is_spoiler = entry_data["spoiler"]
is_viewed = entry_data["clicked"]
upvotes = entry_data["ups"]
downvotes = entry_data["downs"]
comments = entry_data["num_comments"]
except KeyError as e:
raise BuilderMissingDataException(payload=entry) from e
if not rule.reddit_allow_nfsw and is_nsfw:
raise BuilderSkippedException("Rule does not allow NSFW posts")
elif not rule.reddit_allow_spoiler and is_spoiler:
raise BuilderSkippedException("Rule does not allow spoilers")
elif not rule.reddit_allow_viewed and is_viewed:
raise BuilderSkippedException("Post was already seen by user")
elif not upvotes >= rule.reddit_upvotes_min:
raise BuilderSkippedException(
"Post does not meet minimum amount of upvotes"
)
elif (
rule.reddit_downvotes_max is not None
and downvotes > rule.reddit_downvotes_max
):
raise BuilderSkippedException("Post has more downvotes than allowed")
elif not comments >= rule.reddit_comments_min:
raise BuilderSkippedException("Post does not have enough comments")
title = truncate_text(Post, "title", title)
author = truncate_text(Post, "author", author)

1356
src/newsreader/news/collection/tests/reddit/builder/mocks.py
View file

File diff suppressed because it is too large Load diff

90
src/newsreader/news/collection/tests/reddit/builder/tests.py
View file

@ -365,3 +365,93 @@ class RedditBuilderTestCase(TestCase):
builder.save()
self.assertEquals(Post.objects.count(), 0)
def test_nsfw_not_allowed(self):
builder = RedditBuilder
subreddit = SubredditFactory(reddit_allow_nfsw=False)
mock_stream = Mock(rule=subreddit)
with builder(nsfw_mock, mock_stream) as builder:
builder.build()
builder.save()
posts = {post.remote_identifier: post for post in Post.objects.all()}
self.assertEquals(Post.objects.count(), 1)
self.assertCountEqual(("hna75r",), posts.keys())
def test_spoiler_not_allowed(self):
builder = RedditBuilder
subreddit = SubredditFactory(reddit_allow_spoiler=False)
mock_stream = Mock(rule=subreddit)
with builder(spoiler_mock, mock_stream) as builder:
builder.build()
builder.save()
posts = {post.remote_identifier: post for post in Post.objects.all()}
self.assertEquals(Post.objects.count(), 1)
self.assertCountEqual(("hm0qct",), posts.keys())
def test_already_seen_not_allowed(self):
builder = RedditBuilder
subreddit = SubredditFactory(reddit_allow_viewed=False)
mock_stream = Mock(rule=subreddit)
with builder(seen_mock, mock_stream) as builder:
builder.build()
builder.save()
posts = {post.remote_identifier: post for post in Post.objects.all()}
self.assertEquals(Post.objects.count(), 1)
self.assertCountEqual(("hna75r",), posts.keys())
def test_upvote_minimum(self):
builder = RedditBuilder
subreddit = SubredditFactory(reddit_upvotes_min=100)
mock_stream = Mock(rule=subreddit)
with builder(upvote_mock, mock_stream) as builder:
builder.build()
builder.save()
posts = {post.remote_identifier: post for post in Post.objects.all()}
self.assertEquals(Post.objects.count(), 1)
self.assertCountEqual(("hna75r",), posts.keys())
def test_comments_minimum(self):
builder = RedditBuilder
subreddit = SubredditFactory(reddit_comments_min=100)
mock_stream = Mock(rule=subreddit)
with builder(comment_mock, mock_stream) as builder:
builder.build()
builder.save()
posts = {post.remote_identifier: post for post in Post.objects.all()}
self.assertEquals(Post.objects.count(), 1)
self.assertCountEqual(("hm0qct",), posts.keys())
def test_downvote_maximum(self):
builder = RedditBuilder
subreddit = SubredditFactory(reddit_downvotes_max=20)
mock_stream = Mock(rule=subreddit)
with builder(downvote_mock, mock_stream) as builder:
builder.build()
builder.save()
posts = {post.remote_identifier: post for post in Post.objects.all()}
self.assertEquals(Post.objects.count(), 1)
self.assertCountEqual(("hm0qct",), posts.keys())

10
src/newsreader/news/collection/tests/views/test_subreddit_views.py
View file

@ -20,6 +20,11 @@ class SubRedditCreateViewTestCase(CollectionRuleViewTestCase, TestCase):
"name": "new rule",
"url": f"{REDDIT_API_URL}/r/aww",
"category": str(self.category.pk),
"reddit_allow_nfsw": False,
"reddit_allow_spoiler": False,
"reddit_allow_viewed": True,
"reddit_upvotes_min": 0,
"reddit_comments_min": 0,
}
self.url = reverse("news:collection:subreddit-create")
@ -66,6 +71,11 @@ class SubRedditUpdateViewTestCase(CollectionRuleViewTestCase, TestCase):
"url": self.rule.url,
"category": str(self.category.pk),
"timezone": pytz.utc,
"reddit_allow_nfsw": False,
"reddit_allow_spoiler": False,
"reddit_allow_viewed": True,
"reddit_upvotes_min": 0,
"reddit_comments_min": 0,
}
def test_name_change(self):

13
src/newsreader/scss/components/posts/_posts.scss
View file

@ -1,15 +1,16 @@
.posts {
width: 70%;
margin: 0 0 2% 20px;
width: 70%;
&__list {
display: flex;
flex-direction: column;
list-style: none;
width: 95%;
width: 100%;
padding: 0;
list-style: none;
}
&__item {
@ -18,6 +19,8 @@
padding: 10px;
max-width: max-content;
&:first-child {
padding: 0 10px 10px 10px;
}
@ -39,8 +42,6 @@
}
&__header {
width: 80%;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;