diff --git a/docs/explanation/technos.md b/docs/explanation/technos.md index 88e15aa5..542bd95d 100644 --- a/docs/explanation/technos.md +++ b/docs/explanation/technos.md @@ -131,6 +131,31 @@ de données fonctionnent avec l'un comme avec l'autre. Heureusement, et grâce à l'ORM de Django, cette double compatibilité est presque toujours possible. +### Celery + +[Site officiel](https://docs.celeryq.dev/en/stable/) + +Dans certaines situations, on veut séparer une tâche +pour la faire tourner dans son coin. +Deux cas qui correspondent à cette situation sont : + +- les tâches longues à exécuter + (comme l'envoi de mail ou la génération de documents), + pour lesquelles on veut pouvoir dire à l'utilisateur + que sa requête a été prise en compte, sans pour autant + le faire trop patienter +- les tâches régulières séparées du cycle requête/réponse. + +Pour ça, nous utilisons Celery. +Grâce à son intégration avec Django, +il permet de mettre en place une queue de message +avec assez peu complexité ajoutée. + +En outre, ses extensions `django-celery-results` +et `django-celery-beat` enrichissent son intégration +avec django et offrent des moyens de manipuler certaines +tâches directement dans l'interface admin de django. + ## Frontend ### Jinja2 diff --git a/docs/tutorial/install-advanced.md b/docs/tutorial/install-advanced.md index 2998298f..fc3f5ab8 100644 --- a/docs/tutorial/install-advanced.md +++ b/docs/tutorial/install-advanced.md @@ -279,6 +279,20 @@ Toutes les requêtes vers des fichiers statiques et les medias publiques seront servies directement par nginx. Toutes les autres requêtes seront transmises au serveur django. +## Celery + +Celery ne tourne pas dans django. +C'est une application à part, avec ses propres processus, +qui tourne de manière indépendante et qui ne communique +que par messages avec l'instance de django. + +Pour faire tourner Celery, faites la commande suivante dans +un terminal à part : + +```bash +poetry run celery -A sith worker --beat -l INFO +``` + ## Mettre à jour la base de données antispam diff --git a/pyproject.toml b/pyproject.toml index 958a95f9..f8714087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ dependencies = [ "requests>=2.32.3", "honcho>=2.0.0", "psutil>=7.0.0", + "celery[redis]>=5.5.1", + "django-celery-results>=2.5.1", + "django-celery-beat>=2.7.0", ] [project.urls] diff --git a/sith/__init__.py b/sith/__init__.py index f4445e69..1c1f4f81 100644 --- a/sith/__init__.py +++ b/sith/__init__.py @@ -12,3 +12,9 @@ # OR WITHIN THE LOCAL FILE "LICENSE" # # + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/sith/celery.py b/sith/celery.py new file mode 100644 index 00000000..16117fa2 --- /dev/null +++ b/sith/celery.py @@ -0,0 +1,17 @@ +# Set the default Django settings module for the 'celery' program. +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sith.settings") + +app = Celery("sith") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() diff --git a/sith/settings.py b/sith/settings.py index 5ccbd136..05756f05 100644 --- a/sith/settings.py +++ b/sith/settings.py @@ -106,6 +106,8 @@ INSTALLED_APPS = ( "django_jinja", "ninja_extra", "haystack", + "django_celery_results", + "django_celery_beat", "captcha", "core", "club", @@ -336,6 +338,14 @@ EMAIL_BACKEND = env.str( EMAIL_HOST = env.str("EMAIL_HOST", default="localhost") EMAIL_PORT = env.int("EMAIL_PORT", default=25) +# Celery +CELERY_TIMEZONE = TIME_ZONE +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +CElERY_BROKER_URL = env.str("CELERY_BROKER_URL", default="redis://localhost:6379/1") +CELERY_RESULT_BACKEND = "django-db" +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" + # Below this line, only Sith-specific variables are defined SITH_URL = env.str("SITH_URL", default="127.0.0.1:8000") diff --git a/uv.lock b/uv.lock index 3f368e15..13e02547 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/34/d4e1c02d3bee589efb5dfa17f88ea08bdb3e3eac12bc475462aec52ed223/alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92", size = 13511 }, ] +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944 }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -73,6 +85,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/49/6abb616eb3cbab6a7cca303dc02fdf3836de2e0b834bf966a7f5271a34d8/beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16", size = 186015 }, ] +[[package]] +name = "billiard" +version = "4.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/58/1546c970afcd2a2428b1bfafecf2371d8951cc34b46701bea73f4280989e/billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f", size = 155031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/da/43b15f28fe5f9e027b41c539abc5469052e9d48fd75f8ff094ba2a0ae767/billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb", size = 86766 }, +] + [[package]] name = "bracex" version = "2.5.post1" @@ -82,6 +103,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/02/8db98cdc1a58e0abd6716d5e63244658e6e63513c65f469f34b6f1053fd0/bracex-2.5.post1-py3-none-any.whl", hash = "sha256:13e5732fec27828d6af308628285ad358047cec36801598368cb28bc631dbaf6", size = 11558 }, ] +[[package]] +name = "celery" +version = "5.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/7e/a252cc4d003bd98ec350a27f2add5c995862e042a64648b22d4c13ed73cf/celery-5.5.1.tar.gz", hash = "sha256:2af9109a10fe28155044f4c387ce0e5e7f1fc89f9584cfb4b0df94f99a5fedc7", size = 1666151 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/e4/9bc19817230cb88d35ff4dd4415dbb0666cca525875b4c577af47aaf59e9/celery-5.5.1-py3-none-any.whl", hash = "sha256:9f4f9e57e36000c097c1b6f7a8ab29814b82771e5439836f83915823809729c8", size = 438525 }, +] + +[package.optional-dependencies] +redis = [ + { name = "redis" }, +] + [[package]] name = "certifi" version = "2025.1.31" @@ -189,6 +234,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, ] +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631 }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/1d/45434f64ed749540af821fd7e42b8e4d23ac04b1eda7c26613288d6cd8a8/click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", size = 8164 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/da/824b92d9942f4e472702488857914bdd50f73021efea15b4cad9aca8ecef/click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8", size = 7497 }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -246,6 +328,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/f1/4da7717f0063a222db253e7121bd6a56f6fb1ba439dcc36659088793347c/coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", size = 203435 }, ] +[[package]] +name = "cron-descriptor" +version = "1.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/83/70bd410dc6965e33a5460b7da84cf0c5a7330a68d6d5d4c3dfdb72ca117e/cron_descriptor-1.4.5.tar.gz", hash = "sha256:f51ce4ffc1d1f2816939add8524f206c376a42c87a5fca3091ce26725b3b1bca", size = 30666 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/20/2cfe598ead23a715a00beb716477cfddd3e5948cf203c372d02221e5b0c6/cron_descriptor-1.4.5-py3-none-any.whl", hash = "sha256:736b3ae9d1a99bc3dbfc5b55b5e6e7c12031e7ba5de716625772f8b02dcd6013", size = 50370 }, +] + [[package]] name = "cryptography" version = "44.0.2" @@ -353,6 +444,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/90/01755e4a42558b763f7021e9369aa6aa94c2ede7313deed56cb7483834ab/django_cache_url-3.4.5-py2.py3-none-any.whl", hash = "sha256:5f350759978483ab85dc0e3e17b3d53eed3394a28148f6bf0f53d11d0feb5b3c", size = 4760 }, ] +[[package]] +name = "django-celery-beat" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "celery" }, + { name = "cron-descriptor" }, + { name = "django" }, + { name = "django-timezone-field" }, + { name = "python-crontab" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/8f/8a18f234173001bd7a7d63826d2d7f456b38031c892514d27c0f7aea10be/django_celery_beat-2.7.0.tar.gz", hash = "sha256:8482034925e09b698c05ad61c36ed2a8dbc436724a3fe119215193a4ca6dc967", size = 163472 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/f8/f5a25472222b19258c3a53ce71c4efd171a12ab3c988bb3026dec0522a64/django_celery_beat-2.7.0-py3-none-any.whl", hash = "sha256:851c680d8fbf608ca5fecd5836622beea89fa017bc2b3f94a5b8c648c32d84b1", size = 94097 }, +] + +[[package]] +name = "django-celery-results" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "celery" }, + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/24/a13ad6a276f62385da6d83a1995ded452d173841b6e0a83a899c3b75062e/django_celery_results-2.5.1.tar.gz", hash = "sha256:3ecb7147f773f34d0381bac6246337ce4cf88a2ea7b82774ed48e518b67bb8fd", size = 80944 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/2e/aa82857354d5227922c2ae6e83e5214537d8f108184bfeea6704d0b045d8/django_celery_results-2.5.1-py3-none-any.whl", hash = "sha256:0da4cd5ecc049333e4524a23fcfc3460dfae91aa0a60f1fae4b6b2889c254e01", size = 36293 }, +] + [[package]] name = "django-countries" version = "7.6.1" @@ -483,6 +604,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/63/7485d3049c8479f2c77c87aa7557aea300289ad7ac95c9230759f6610c6f/django_simple_captcha-0.6.2-py2.py3-none-any.whl", hash = "sha256:f5b7eee6bfeba6c55b47f2f8414557d5609fc29c4758da2c91ba9a91d6ac349d", size = 93578 }, ] +[[package]] +name = "django-timezone-field" +version = "7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/5b/0dbe271fef3c2274b83dbcb1b19fa3dacf1f7e542382819294644e78ea8b/django_timezone_field-7.1.tar.gz", hash = "sha256:b3ef409d88a2718b566fabe10ea996f2838bc72b22d3a2900c0aa905c761380c", size = 13727 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/09/7a808392a751a24ffa62bec00e3085a9c1a151d728c323a5bab229ea0e58/django_timezone_field-7.1-py3-none-any.whl", hash = "sha256:93914713ed882f5bccda080eda388f7006349f25930b6122e9b07bf8db49c4b4", size = 13177 }, +] + [[package]] name = "djhtml" version = "3.0.7" @@ -751,6 +884,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, ] +[[package]] +name = "kombu" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/12/7a340f48920f30d6febb65d0c4aca70ed01b29e116131152977df78a9a39/kombu-5.5.2.tar.gz", hash = "sha256:2dd27ec84fd843a4e0a7187424313f87514b344812cb98c25daddafbb6a7ff0e", size = 461522 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/ba/939f3db0fca87715c883e42cc93045347d61a9d519c270a38e54a06db6e1/kombu-5.5.2-py3-none-any.whl", hash = "sha256:40f3674ed19603b8a771b6c74de126dbf8879755a0337caac6602faa82d539cd", size = 209763 }, +] + [[package]] name = "libsass" version = "0.23.0" @@ -1391,6 +1538,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/ac/bd0608d229ec808e51a21044f3f2f27b9a37e7a0ebaca7247882e67876af/pytest_django-4.11.1-py3-none-any.whl", hash = "sha256:1b63773f648aa3d8541000c26929c1ea63934be1cfa674c76436966d73fe6a10", size = 25281 }, ] +[[package]] +name = "python-crontab" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/f0/25775565c133d4e29eeb607bf9ddba0075f3af36041a1844dd207881047f/python_crontab-3.2.0.tar.gz", hash = "sha256:40067d1dd39ade3460b2ad8557c7651514cd3851deffff61c5c60e1227c5c36b", size = 57001 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/91/832fb3b3a1f62bd2ab4924f6be0c7736c9bc4f84d3b153b74efcf6d4e4a1/python_crontab-3.2.0-py3-none-any.whl", hash = "sha256:82cb9b6a312d41ff66fd3caf3eed7115c28c195bfb50711bc2b4b9592feb9fe5", size = 27351 }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1561,9 +1720,12 @@ name = "sith" version = "3" source = { editable = "." } dependencies = [ + { name = "celery", extra = ["redis"] }, { name = "cryptography" }, { name = "dict2xml" }, { name = "django" }, + { name = "django-celery-beat" }, + { name = "django-celery-results" }, { name = "django-countries" }, { name = "django-haystack" }, { name = "django-honeypot" }, @@ -1625,9 +1787,12 @@ tests = [ [package.metadata] requires-dist = [ + { name = "celery", extras = ["redis"], specifier = ">=5.5.1" }, { name = "cryptography", specifier = ">=44.0.2,<45.0.0" }, { name = "dict2xml", specifier = ">=1.7.6,<2.0.0" }, { name = "django", specifier = ">=5.2.0,<6.0.0" }, + { name = "django-celery-beat", specifier = ">=2.7.0" }, + { name = "django-celery-results", specifier = ">=2.5.1" }, { name = "django-countries", specifier = ">=7.6.1,<8.0.0" }, { name = "django-haystack", specifier = ">=3.3.0,<4.0.0" }, { name = "django-honeypot", git = "https://github.com/jamesturk/django-honeypot.git?rev=3986228" }, @@ -1893,6 +2058,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813", size = 128680 }, ] +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636 }, +] + [[package]] name = "virtualenv" version = "20.30.0"