From 9edc88c38488c0322fd3e9aa7a00d5180553a726 Mon Sep 17 00:00:00 2001 From: imperosol Date: Mon, 23 Dec 2024 19:40:41 +0100 Subject: [PATCH] celery --- .env.example | 2 + docs/explanation/technos.md | 25 +++++ docs/tutorial/install-advanced.md | 14 +++ pyproject.toml | 3 + sith/__init__.py | 6 + sith/celery.py | 17 +++ sith/settings.py | 14 ++- uv.lock | 175 ++++++++++++++++++++++++++++++ 8 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 sith/celery.py diff --git a/.env.example b/.env.example index 266af2a1..bab595ff 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,8 @@ DATABASE_URL=sqlite:///db.sqlite3 #DATABASE_URL=postgres://user:password@127.0.0.1:5432/sith CACHE_URL=redis://127.0.0.1:6379/0 +CELERY_BROKER_URL=redis://127.0.0.1:6379/1 + MEDIA_ROOT=data STATIC_ROOT=static diff --git a/docs/explanation/technos.md b/docs/explanation/technos.md index 88e15aa5..542bd95d 100644 --- a/docs/explanation/technos.md +++ b/docs/explanation/technos.md @@ -131,6 +131,31 @@ de données fonctionnent avec l'un comme avec l'autre. Heureusement, et grâce à l'ORM de Django, cette double compatibilité est presque toujours possible. +### Celery + +[Site officiel](https://docs.celeryq.dev/en/stable/) + +Dans certaines situations, on veut séparer une tâche +pour la faire tourner dans son coin. +Deux cas qui correspondent à cette situation sont : + +- les tâches longues à exécuter + (comme l'envoi de mail ou la génération de documents), + pour lesquelles on veut pouvoir dire à l'utilisateur + que sa requête a été prise en compte, sans pour autant + le faire trop patienter +- les tâches régulières séparées du cycle requête/réponse. + +Pour ça, nous utilisons Celery. +Grâce à son intégration avec Django, +il permet de mettre en place une queue de message +avec assez peu complexité ajoutée. + +En outre, ses extensions `django-celery-results` +et `django-celery-beat` enrichissent son intégration +avec django et offrent des moyens de manipuler certaines +tâches directement dans l'interface admin de django. + ## Frontend ### Jinja2 diff --git a/docs/tutorial/install-advanced.md b/docs/tutorial/install-advanced.md index 357d013c..e2d3aa5e 100644 --- a/docs/tutorial/install-advanced.md +++ b/docs/tutorial/install-advanced.md @@ -227,6 +227,20 @@ Toutes les requêtes vers des fichiers statiques et les medias publiques seront seront servies directement par nginx. Toutes les autres requêtes seront transmises au serveur django. +## Celery + +Celery ne tourne pas dans django. +C'est une application à part, avec ses propres processus, +qui tourne de manière indépendante et qui ne communique +que par messages avec l'instance de django. + +Pour faire tourner Celery, faites la commande suivante dans +un terminal à part : + +```bash +poetry run celery -A sith worker --beat -l INFO +``` + ## Mettre à jour la base de données antispam diff --git a/pyproject.toml b/pyproject.toml index 9b0a5a81..40c579b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,9 @@ dependencies = [ "ical<9.0.0,>=8.3.0", "redis[hiredis]<6.0.0,>=5.2.0", "environs[django]<15.0.0,>=14.1.0", + "celery[redis]<6.0.0,>=5.4.0", + "django-celery-results<3.0.0,>=2.5.1", + "django-celery-beat<3.0.0,>=2.7.0", ] [project.urls] diff --git a/sith/__init__.py b/sith/__init__.py index f4445e69..1c1f4f81 100644 --- a/sith/__init__.py +++ b/sith/__init__.py @@ -12,3 +12,9 @@ # OR WITHIN THE LOCAL FILE "LICENSE" # # + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/sith/celery.py b/sith/celery.py new file mode 100644 index 00000000..16117fa2 --- /dev/null +++ b/sith/celery.py @@ -0,0 +1,17 @@ +# Set the default Django settings module for the 'celery' program. +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sith.settings") + +app = Celery("sith") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() diff --git a/sith/settings.py b/sith/settings.py index 598e1fb6..1e20e982 100644 --- a/sith/settings.py +++ b/sith/settings.py @@ -88,6 +88,8 @@ INSTALLED_APPS = ( "django_jinja", "ninja_extra", "haystack", + "django_celery_results", + "django_celery_beat", "captcha", "core", "club", @@ -271,13 +273,13 @@ PHONENUMBER_DEFAULT_REGION = "FR" # Medias MEDIA_URL = "/data/" -MEDIA_ROOT = env.path("MEDIA_ROOT", default="data") +MEDIA_ROOT = env.path("MEDIA_ROOT", default=BASE_DIR / "data") # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.8/howto/static-files/ STATIC_URL = "/static/" -STATIC_ROOT = env.path("STATIC_ROOT", default="static") +STATIC_ROOT = env.path("STATIC_ROOT", default=BASE_DIR / "static") # Static files finders which allow to see static folder in all apps STATICFILES_FINDERS = [ @@ -319,6 +321,14 @@ EMAIL_BACKEND = env.str( EMAIL_HOST = env.str("EMAIL_HOST", default="localhost") EMAIL_PORT = env.int("EMAIL_PORT", default=25) +# Celery +CELERY_TIMEZONE = TIME_ZONE +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +CElERY_BROKER_URL = env.str("CELERY_BROKER_URL", default="redis://localhost:6379/1") +CELERY_RESULT_BACKEND = "django-db" +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" + # Below this line, only Sith-specific variables are defined SITH_URL = env.str("SITH_URL", default="127.0.0.1:8000") diff --git a/uv.lock b/uv.lock index 441b0279..1d146c2b 100644 --- a/uv.lock +++ b/uv.lock @@ -10,6 +10,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/34/d4e1c02d3bee589efb5dfa17f88ea08bdb3e3eac12bc475462aec52ed223/alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92", size = 13511 }, ] +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944 }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -46,6 +58,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/20/bc79bc575ba2e2a7f70e8a1155618bb1301eaa5132a8271373a6903f73f8/babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b", size = 9587599 }, ] +[[package]] +name = "billiard" +version = "4.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/58/1546c970afcd2a2428b1bfafecf2371d8951cc34b46701bea73f4280989e/billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f", size = 155031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/da/43b15f28fe5f9e027b41c539abc5469052e9d48fd75f8ff094ba2a0ae767/billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb", size = 86766 }, +] + [[package]] name = "bracex" version = "2.5.post1" @@ -55,6 +76,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/02/8db98cdc1a58e0abd6716d5e63244658e6e63513c65f469f34b6f1053fd0/bracex-2.5.post1-py3-none-any.whl", hash = "sha256:13e5732fec27828d6af308628285ad358047cec36801598368cb28bc631dbaf6", size = 11558 }, ] +[[package]] +name = "celery" +version = "5.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/9c/cf0bce2cc1c8971bf56629d8f180e4ca35612c7e79e6e432e785261a8be4/celery-5.4.0.tar.gz", hash = "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706", size = 1575692 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/c4/6a4d3772e5407622feb93dd25c86ce3c0fee746fa822a777a627d56b4f2a/celery-5.4.0-py3-none-any.whl", hash = "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64", size = 425983 }, +] + +[package.optional-dependencies] +redis = [ + { name = "redis" }, +] + [[package]] name = "certifi" version = "2024.12.14" @@ -162,6 +208,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, ] +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631 }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/1d/45434f64ed749540af821fd7e42b8e4d23ac04b1eda7c26613288d6cd8a8/click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", size = 8164 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/da/824b92d9942f4e472702488857914bdd50f73021efea15b4cad9aca8ecef/click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8", size = 7497 }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -218,6 +301,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/55/52f5e66142a9d7bc93a15192eba7a78513d2abf6b3558d77b4ca32f5f424/coverage-7.6.10-cp313-cp313t-win_amd64.whl", hash = "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d", size = 212781 }, ] +[[package]] +name = "cron-descriptor" +version = "1.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/83/70bd410dc6965e33a5460b7da84cf0c5a7330a68d6d5d4c3dfdb72ca117e/cron_descriptor-1.4.5.tar.gz", hash = "sha256:f51ce4ffc1d1f2816939add8524f206c376a42c87a5fca3091ce26725b3b1bca", size = 30666 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/20/2cfe598ead23a715a00beb716477cfddd3e5948cf203c372d02221e5b0c6/cron_descriptor-1.4.5-py3-none-any.whl", hash = "sha256:736b3ae9d1a99bc3dbfc5b55b5e6e7c12031e7ba5de716625772f8b02dcd6013", size = 50370 }, +] + [[package]] name = "cryptography" version = "44.0.0" @@ -321,6 +413,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/90/01755e4a42558b763f7021e9369aa6aa94c2ede7313deed56cb7483834ab/django_cache_url-3.4.5-py2.py3-none-any.whl", hash = "sha256:5f350759978483ab85dc0e3e17b3d53eed3394a28148f6bf0f53d11d0feb5b3c", size = 4760 }, ] +[[package]] +name = "django-celery-beat" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "celery" }, + { name = "cron-descriptor" }, + { name = "django" }, + { name = "django-timezone-field" }, + { name = "python-crontab" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/8f/8a18f234173001bd7a7d63826d2d7f456b38031c892514d27c0f7aea10be/django_celery_beat-2.7.0.tar.gz", hash = "sha256:8482034925e09b698c05ad61c36ed2a8dbc436724a3fe119215193a4ca6dc967", size = 163472 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/f8/f5a25472222b19258c3a53ce71c4efd171a12ab3c988bb3026dec0522a64/django_celery_beat-2.7.0-py3-none-any.whl", hash = "sha256:851c680d8fbf608ca5fecd5836622beea89fa017bc2b3f94a5b8c648c32d84b1", size = 94097 }, +] + +[[package]] +name = "django-celery-results" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "celery" }, + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/24/a13ad6a276f62385da6d83a1995ded452d173841b6e0a83a899c3b75062e/django_celery_results-2.5.1.tar.gz", hash = "sha256:3ecb7147f773f34d0381bac6246337ce4cf88a2ea7b82774ed48e518b67bb8fd", size = 80944 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/2e/aa82857354d5227922c2ae6e83e5214537d8f108184bfeea6704d0b045d8/django_celery_results-2.5.1-py3-none-any.whl", hash = "sha256:0da4cd5ecc049333e4524a23fcfc3460dfae91aa0a60f1fae4b6b2889c254e01", size = 36293 }, +] + [[package]] name = "django-countries" version = "7.6.1" @@ -455,6 +577,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/fa/83160be2a72b98ebc04405351cb46a8e74c4468b0bcbdeceece266b94ca7/django_simple_captcha-0.6.0-py2.py3-none-any.whl", hash = "sha256:3ae9a7e650cb0cdbcfd4a75aa91fdf25dcc523ef541a7b1f004bd4357798fc03", size = 92268 }, ] +[[package]] +name = "django-timezone-field" +version = "7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/b3/992aa517b95f2e6934aa05b8160cf55f91c49c7b91e33076ea9af2f29920/django_timezone_field-7.0.tar.gz", hash = "sha256:aa6f4965838484317b7f08d22c0d91a53d64e7bbbd34264468ae83d4023898a7", size = 13683 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/f9/11769c4414026f1a9ce3e581731d07b084683fc7b4c580703dc71ef81347/django_timezone_field-7.0-py3-none-any.whl", hash = "sha256:3232e7ecde66ba4464abb6f9e6b8cc739b914efb9b29dc2cf2eee451f7cc2acb", size = 13161 }, +] + [[package]] name = "djhtml" version = "3.0.7" @@ -699,6 +833,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/0f/2ba5fbcd631e3e88689309dbe978c5769e883e4b84ebfe7da30b43275c5a/jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb", size = 134596 }, ] +[[package]] +name = "kombu" +version = "5.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/4d/b93fcb353d279839cc35d0012bee805ed0cf61c07587916bfc35dbfddaf1/kombu-5.4.2.tar.gz", hash = "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf", size = 442858 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ec/7811a3cf9fdfee3ee88e54d08fcbc3fabe7c1b6e4059826c59d7b795651c/kombu-5.4.2-py3-none-any.whl", hash = "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763", size = 201349 }, +] + [[package]] name = "libsass" version = "0.23.0" @@ -1280,6 +1428,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/fe/54f387ee1b41c9ad59e48fb8368a361fad0600fe404315e31a12bacaea7d/pytest_django-4.9.0-py3-none-any.whl", hash = "sha256:1d83692cb39188682dbb419ff0393867e9904094a549a7d38a3154d5731b2b99", size = 23723 }, ] +[[package]] +name = "python-crontab" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/f0/25775565c133d4e29eeb607bf9ddba0075f3af36041a1844dd207881047f/python_crontab-3.2.0.tar.gz", hash = "sha256:40067d1dd39ade3460b2ad8557c7651514cd3851deffff61c5c60e1227c5c36b", size = 57001 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/91/832fb3b3a1f62bd2ab4924f6be0c7736c9bc4f84d3b153b74efcf6d4e4a1/python_crontab-3.2.0-py3-none-any.whl", hash = "sha256:82cb9b6a312d41ff66fd3caf3eed7115c28c195bfb50711bc2b4b9592feb9fe5", size = 27351 }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1488,9 +1648,12 @@ name = "sith" version = "3" source = { virtual = "." } dependencies = [ + { name = "celery", extra = ["redis"] }, { name = "cryptography" }, { name = "dict2xml" }, { name = "django" }, + { name = "django-celery-beat" }, + { name = "django-celery-results" }, { name = "django-countries" }, { name = "django-haystack" }, { name = "django-honeypot" }, @@ -1547,9 +1710,12 @@ tests = [ [package.metadata] requires-dist = [ + { name = "celery", extras = ["redis"], specifier = ">=5.4.0,<6.0.0" }, { name = "cryptography", specifier = ">=44.0.0,<45.0.0" }, { name = "dict2xml", specifier = ">=1.7.6,<2.0.0" }, { name = "django", specifier = ">=4.2.17,<5.0.0" }, + { name = "django-celery-beat", specifier = ">=2.7.0,<3.0.0" }, + { name = "django-celery-results", specifier = ">=2.5.1,<3.0.0" }, { name = "django-countries", specifier = ">=7.6.1,<8.0.0" }, { name = "django-haystack", specifier = ">=3.3.0,<4.0.0" }, { name = "django-honeypot", specifier = ">=1.2.1,<2.0.0" }, @@ -1789,6 +1955,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 }, ] +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636 }, +] + [[package]] name = "virtualenv" version = "20.28.1"