diff --git a/docker-compose.yml b/docker-compose.yml index d400e30..2a06653 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,14 +51,14 @@ services: environment: <<: *django-env command: | - celery worker -n worker1@%h - -n worker2@%h - --app newsreader - --loglevel INFO - --concurrency 2 + celery --app newsreader --workdir /app/src/ + worker --loglevel INFO + --concurrency 2 --beat --scheduler django + -n worker1@%h + -n worker2@%h depends_on: rabbitmq: condition: service_started diff --git a/pyproject.toml b/pyproject.toml index d51b36b..bad9050 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,11 @@ license = {text = 'GPL-3.0'} requires-python = '>=3.11' dependencies = [ 'django~=3.2', - 'celery~=4.4', + 'celery~=5.0', 'psycopg2', 'django-axes', - 'django-celery-beat~=1.6.0', + 'django-celery-beat~=2.5.0', 'django-registration-redux~=2.7', 'django-rest-framework', 'drf-yasg', diff --git a/requirements/base.txt b/requirements/base.txt index 190d9c5..53d38de 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,17 +4,17 @@ # # pip-compile --output-file=requirements/base.txt --resolver=backtracking pyproject.toml # -amqp==2.6.1 +amqp==5.1.1 # via kombu asgiref==3.7.2 # via django beautifulsoup4==4.12.2 # via newsreader (pyproject.toml) -billiard==3.6.4.0 +billiard==4.1.0 # via celery bleach==6.0.0 # via newsreader (pyproject.toml) -celery==4.4.7 +celery==5.3.1 # via # django-celery-beat # newsreader (pyproject.toml) @@ -22,6 +22,20 @@ certifi==2023.5.7 # via requests charset-normalizer==3.1.0 # via requests +click==8.1.3 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery +cron-descriptor==1.4.0 + # via django-celery-beat django==3.2.19 # via # django-axes @@ -32,13 +46,13 @@ django==3.2.19 # newsreader (pyproject.toml) django-axes==6.0.4 # via newsreader (pyproject.toml) -django-celery-beat==1.6.0 +django-celery-beat==2.5.0 # via newsreader (pyproject.toml) django-registration-redux==2.12 # via newsreader (pyproject.toml) django-rest-framework==0.1.0 # via newsreader (pyproject.toml) -django-timezone-field==3.1 +django-timezone-field==5.1 # via django-celery-beat djangorestframework==3.14.0 # via @@ -54,7 +68,7 @@ idna==3.4 # via requests inflection==0.5.1 # via drf-yasg -kombu==4.6.11 +kombu==5.3.1 # via celery lxml==4.9.2 # via newsreader (pyproject.toml) @@ -62,19 +76,22 @@ oauthlib==3.2.2 # via requests-oauthlib packaging==23.1 # via drf-yasg +prompt-toolkit==3.0.38 + # via click-repl psycopg2==2.9.6 # via newsreader (pyproject.toml) python-crontab==2.7.1 # via django-celery-beat python-dateutil==2.8.2 - # via python-crontab + # via + # celery + # python-crontab python-dotenv==0.21.1 # via newsreader (pyproject.toml) python-memcached==1.59 # via newsreader (pyproject.toml) pytz==2023.3 # via - # celery # django # django-timezone-field # djangorestframework @@ -99,17 +116,26 @@ soupsieve==2.4.1 sqlparse==0.4.4 # via django typing-extensions==4.6.3 - # via asgiref + # via + # asgiref + # kombu +tzdata==2023.3 + # via + # celery + # django-celery-beat uritemplate==4.1.1 # via drf-yasg urllib3==2.0.3 # via requests -vine==1.3.0 +vine==5.0.0 # via # amqp # celery + # kombu wcwidth==0.2.6 - # via ftfy + # via + # ftfy + # prompt-toolkit webencodings==0.5.1 # via bleach diff --git a/requirements/ci.txt b/requirements/ci.txt index 32335e3..4e2403e 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=ci --extra=testing --output-file=requirements/ci.txt --resolver=backtracking pyproject.toml requirements/base.txt requirements/testing.txt # -amqp==2.6.1 +amqp==5.1.1 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -22,7 +22,7 @@ beautifulsoup4==4.12.2 # via # -r requirements/base.txt # newsreader (pyproject.toml) -billiard==3.6.4.0 +billiard==4.1.0 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -35,7 +35,7 @@ bleach==6.0.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -celery==4.4.7 +celery==5.3.1 # via # -r requirements/base.txt # django-celery-beat @@ -52,10 +52,35 @@ charset-normalizer==3.1.0 # requests click==8.1.3 # via + # -r requirements/base.txt # -r requirements/testing.txt # black + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery coverage==7.2.7 # via newsreader (pyproject.toml) +cron-descriptor==1.4.0 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # django-celery-beat django==3.2.19 # via # -r requirements/base.txt @@ -69,7 +94,7 @@ django-axes==6.0.4 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-celery-beat==1.6.0 +django-celery-beat==2.5.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) @@ -81,7 +106,7 @@ django-rest-framework==0.1.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-timezone-field==3.1 +django-timezone-field==5.1 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -130,7 +155,7 @@ isort==5.12.0 # via # -r requirements/testing.txt # newsreader (pyproject.toml) -kombu==4.6.11 +kombu==5.3.1 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -162,6 +187,11 @@ platformdirs==3.8.0 # via # -r requirements/testing.txt # black +prompt-toolkit==3.0.38 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # click-repl psycopg2==2.9.6 # via # -r requirements/base.txt @@ -179,6 +209,7 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # -r requirements/testing.txt + # celery # faker # freezegun # python-crontab @@ -194,7 +225,6 @@ pytz==2023.3 # via # -r requirements/base.txt # -r requirements/testing.txt - # celery # django # django-timezone-field # djangorestframework @@ -250,6 +280,13 @@ typing-extensions==4.6.3 # -r requirements/testing.txt # asgiref # black + # kombu +tzdata==2023.3 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery + # django-celery-beat uritemplate==4.1.1 # via # -r requirements/base.txt @@ -260,17 +297,19 @@ urllib3==2.0.3 # -r requirements/base.txt # -r requirements/testing.txt # requests -vine==1.3.0 +vine==5.0.0 # via # -r requirements/base.txt # -r requirements/testing.txt # amqp # celery + # kombu wcwidth==0.2.6 # via # -r requirements/base.txt # -r requirements/testing.txt # ftfy + # prompt-toolkit webencodings==0.5.1 # via # -r requirements/base.txt diff --git a/requirements/development.txt b/requirements/development.txt index b18f83c..a77dd44 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=development --extra=testing --output-file=requirements/development.txt --resolver=backtracking pyproject.toml requirements/base.txt requirements/testing.txt # -amqp==2.6.1 +amqp==5.1.1 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -22,7 +22,7 @@ beautifulsoup4==4.12.2 # via # -r requirements/base.txt # newsreader (pyproject.toml) -billiard==3.6.4.0 +billiard==4.1.0 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -37,7 +37,7 @@ bleach==6.0.0 # newsreader (pyproject.toml) build==0.10.0 # via pip-tools -celery==4.4.7 +celery==5.3.1 # via # -r requirements/base.txt # django-celery-beat @@ -54,9 +54,34 @@ charset-normalizer==3.1.0 # requests click==8.1.3 # via + # -r requirements/base.txt # -r requirements/testing.txt # black + # celery + # click-didyoumean + # click-plugins + # click-repl # pip-tools +click-didyoumean==0.3.0 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery +cron-descriptor==1.4.0 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # django-celery-beat django==3.2.19 # via # -r requirements/base.txt @@ -72,7 +97,7 @@ django-axes==6.0.4 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-celery-beat==1.6.0 +django-celery-beat==2.5.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) @@ -88,7 +113,7 @@ django-rest-framework==0.1.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-timezone-field==3.1 +django-timezone-field==5.1 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -137,7 +162,7 @@ isort==5.12.0 # via # -r requirements/testing.txt # newsreader (pyproject.toml) -kombu==4.6.11 +kombu==5.3.1 # via # -r requirements/base.txt # -r requirements/testing.txt @@ -172,6 +197,11 @@ platformdirs==3.8.0 # via # -r requirements/testing.txt # black +prompt-toolkit==3.0.38 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # click-repl psycopg2==2.9.6 # via # -r requirements/base.txt @@ -191,6 +221,7 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # -r requirements/testing.txt + # celery # faker # freezegun # python-crontab @@ -206,7 +237,6 @@ pytz==2023.3 # via # -r requirements/base.txt # -r requirements/testing.txt - # celery # django # django-timezone-field # djangorestframework @@ -265,6 +295,13 @@ typing-extensions==4.6.3 # -r requirements/testing.txt # asgiref # black + # kombu +tzdata==2023.3 + # via + # -r requirements/base.txt + # -r requirements/testing.txt + # celery + # django-celery-beat uritemplate==4.1.1 # via # -r requirements/base.txt @@ -275,17 +312,19 @@ urllib3==2.0.3 # -r requirements/base.txt # -r requirements/testing.txt # requests -vine==1.3.0 +vine==5.0.0 # via # -r requirements/base.txt # -r requirements/testing.txt # amqp # celery + # kombu wcwidth==0.2.6 # via # -r requirements/base.txt # -r requirements/testing.txt # ftfy + # prompt-toolkit webencodings==0.5.1 # via # -r requirements/base.txt diff --git a/requirements/production.txt b/requirements/production.txt index 752d5ba..cf0aa06 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=production --output-file=requirements/production.txt --resolver=backtracking pyproject.toml requirements/base.txt # -amqp==2.6.1 +amqp==5.1.1 # via # -r requirements/base.txt # kombu @@ -16,7 +16,7 @@ beautifulsoup4==4.12.2 # via # -r requirements/base.txt # newsreader (pyproject.toml) -billiard==3.6.4.0 +billiard==4.1.0 # via # -r requirements/base.txt # celery @@ -24,7 +24,7 @@ bleach==6.0.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -celery==4.4.7 +celery==5.3.1 # via # -r requirements/base.txt # django-celery-beat @@ -38,6 +38,29 @@ charset-normalizer==3.1.0 # via # -r requirements/base.txt # requests +click==8.1.3 + # via + # -r requirements/base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via + # -r requirements/base.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/base.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/base.txt + # celery +cron-descriptor==1.4.0 + # via + # -r requirements/base.txt + # django-celery-beat django==3.2.19 # via # -r requirements/base.txt @@ -51,7 +74,7 @@ django-axes==6.0.4 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-celery-beat==1.6.0 +django-celery-beat==2.5.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) @@ -63,7 +86,7 @@ django-rest-framework==0.1.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-timezone-field==3.1 +django-timezone-field==5.1 # via # -r requirements/base.txt # django-celery-beat @@ -94,7 +117,7 @@ inflection==0.5.1 # via # -r requirements/base.txt # drf-yasg -kombu==4.6.11 +kombu==5.3.1 # via # -r requirements/base.txt # celery @@ -110,6 +133,10 @@ packaging==23.1 # via # -r requirements/base.txt # drf-yasg +prompt-toolkit==3.0.38 + # via + # -r requirements/base.txt + # click-repl psycopg2==2.9.6 # via # -r requirements/base.txt @@ -121,6 +148,7 @@ python-crontab==2.7.1 python-dateutil==2.8.2 # via # -r requirements/base.txt + # celery # python-crontab python-dotenv==0.21.1 # via @@ -133,7 +161,6 @@ python-memcached==1.59 pytz==2023.3 # via # -r requirements/base.txt - # celery # django # django-timezone-field # djangorestframework @@ -175,6 +202,12 @@ typing-extensions==4.6.3 # via # -r requirements/base.txt # asgiref + # kombu +tzdata==2023.3 + # via + # -r requirements/base.txt + # celery + # django-celery-beat uritemplate==4.1.1 # via # -r requirements/base.txt @@ -184,15 +217,17 @@ urllib3==2.0.3 # -r requirements/base.txt # requests # sentry-sdk -vine==1.3.0 +vine==5.0.0 # via # -r requirements/base.txt # amqp # celery + # kombu wcwidth==0.2.6 # via # -r requirements/base.txt # ftfy + # prompt-toolkit webencodings==0.5.1 # via # -r requirements/base.txt diff --git a/requirements/testing.txt b/requirements/testing.txt index e260a7d..96ddce9 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=testing --output-file=requirements/testing.txt --resolver=backtracking pyproject.toml requirements/base.txt # -amqp==2.6.1 +amqp==5.1.1 # via # -r requirements/base.txt # kombu @@ -18,7 +18,7 @@ beautifulsoup4==4.12.2 # via # -r requirements/base.txt # newsreader (pyproject.toml) -billiard==3.6.4.0 +billiard==4.1.0 # via # -r requirements/base.txt # celery @@ -28,7 +28,7 @@ bleach==6.0.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -celery==4.4.7 +celery==5.3.1 # via # -r requirements/base.txt # django-celery-beat @@ -42,7 +42,29 @@ charset-normalizer==3.1.0 # -r requirements/base.txt # requests click==8.1.3 - # via black + # via + # -r requirements/base.txt + # black + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via + # -r requirements/base.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/base.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/base.txt + # celery +cron-descriptor==1.4.0 + # via + # -r requirements/base.txt + # django-celery-beat django==3.2.19 # via # -r requirements/base.txt @@ -56,7 +78,7 @@ django-axes==6.0.4 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-celery-beat==1.6.0 +django-celery-beat==2.5.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) @@ -68,7 +90,7 @@ django-rest-framework==0.1.0 # via # -r requirements/base.txt # newsreader (pyproject.toml) -django-timezone-field==3.1 +django-timezone-field==5.1 # via # -r requirements/base.txt # django-celery-beat @@ -105,7 +127,7 @@ inflection==0.5.1 # drf-yasg isort==5.12.0 # via newsreader (pyproject.toml) -kombu==4.6.11 +kombu==5.3.1 # via # -r requirements/base.txt # celery @@ -128,6 +150,10 @@ pathspec==0.11.1 # via black platformdirs==3.8.0 # via black +prompt-toolkit==3.0.38 + # via + # -r requirements/base.txt + # click-repl psycopg2==2.9.6 # via # -r requirements/base.txt @@ -141,6 +167,7 @@ python-crontab==2.7.1 python-dateutil==2.8.2 # via # -r requirements/base.txt + # celery # faker # freezegun # python-crontab @@ -155,7 +182,6 @@ python-memcached==1.59 pytz==2023.3 # via # -r requirements/base.txt - # celery # django # django-timezone-field # djangorestframework @@ -202,6 +228,12 @@ typing-extensions==4.6.3 # -r requirements/base.txt # asgiref # black + # kombu +tzdata==2023.3 + # via + # -r requirements/base.txt + # celery + # django-celery-beat uritemplate==4.1.1 # via # -r requirements/base.txt @@ -210,15 +242,17 @@ urllib3==2.0.3 # via # -r requirements/base.txt # requests -vine==1.3.0 +vine==5.0.0 # via # -r requirements/base.txt # amqp # celery + # kombu wcwidth==0.2.6 # via # -r requirements/base.txt # ftfy + # prompt-toolkit webencodings==0.5.1 # via # -r requirements/base.txt diff --git a/src/newsreader/utils/celery.py b/src/newsreader/utils/celery.py index 84572c6..4a03e71 100644 --- a/src/newsreader/utils/celery.py +++ b/src/newsreader/utils/celery.py @@ -1,6 +1,6 @@ -from django.core.cache import cache +from time import monotonic -from celery.five import monotonic +from django.core.cache import cache LOCK_EXPIRE = 60 * 10 # 10 minutes