Merge pull request #756 from ae-utbm/antispam

Filter blocked emails

.gitignore

@@ -1,4 +1,4 @@
-db.sqlite3
+*.sqlite3
 *.log
 *.pyc
 *.mo

antispam/admin.py

@@ -0,0 +1,10 @@
+from django.contrib import admin
+
+from antispam.models import ToxicDomain
+
+
+@admin.register(ToxicDomain)
+class ToxicDomainAdmin(admin.ModelAdmin):
+    list_display = ("domain", "is_externally_managed", "created")
+    search_fields = ("domain", "is_externally_managed", "created")
+    list_filter = ("is_externally_managed",)

antispam/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class AntispamConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    verbose_name = "antispam"
+    name = "antispam"

antispam/forms.py

@@ -0,0 +1,18 @@
+import re
+
+from django import forms
+from django.core.validators import EmailValidator
+from django.utils.translation import gettext_lazy as _
+
+from antispam.models import ToxicDomain
+
+
+class AntiSpamEmailField(forms.EmailField):
+    """An email field that email addresses with a known toxic domain."""
+
+    def run_validators(self, value: str):
+        super().run_validators(value)
+        # Domain part should exist since email validation is guaranteed to run first
+        domain = re.search(EmailValidator.domain_regex, value)
+        if ToxicDomain.objects.filter(domain=domain[0]).exists():
+            raise forms.ValidationError(_("Email domain is not allowed."))

antispam/management/commands/update_spam_database.py

@@ -0,0 +1,69 @@
+import requests
+from django.conf import settings
+from django.core.management import BaseCommand
+from django.db.models import Max
+from django.utils import timezone
+
+from antispam.models import ToxicDomain
+
+
+class Command(BaseCommand):
+    """Update blocked ips/mails database"""
+
+    help = "Update blocked ips/mails database"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--force", action="store_true", help="Force re-creation even if up to date"
+        )
+
+    def _should_update(self, *, force: bool = False) -> bool:
+        if force:
+            return True
+        oldest = ToxicDomain.objects.filter(is_externally_managed=True).aggregate(
+            res=Max("created")
+        )["res"]
+        return not (oldest and timezone.now() < (oldest + timezone.timedelta(days=1)))
+
+    def _download_domains(self, providers: list[str]) -> set[str]:
+        domains = set()
+        for provider in providers:
+            res = requests.get(provider)
+            if not res.ok:
+                self.stderr.write(
+                    f"Source {provider} responded with code {res.status_code}"
+                )
+                continue
+            domains |= set(res.content.decode().splitlines())
+        return domains
+
+    def _update_domains(self, domains: set[str]):
+        # Cleanup database
+        ToxicDomain.objects.filter(is_externally_managed=True).delete()
+
+        # Create database
+        ToxicDomain.objects.bulk_create(
+            [
+                ToxicDomain(domain=domain, is_externally_managed=True)
+                for domain in domains
+            ],
+            ignore_conflicts=True,
+        )
+        self.stdout.write("Domain database updated")
+
+    def handle(self, *args, **options):
+        if not self._should_update(force=options["force"]):
+            self.stdout.write("Domain database is up to date")
+            return
+        self.stdout.write("Updating domain database")
+
+        domains = self._download_domains(settings.TOXIC_DOMAINS_PROVIDERS)
+
+        if not domains:
+            self.stderr.write(
+                "No domains could be fetched from settings.TOXIC_DOMAINS_PROVIDERS. "
+                "Please, have a look at your settings."
+            )
+            return
+
+        self._update_domains(domains)

antispam/migrations/0001_initial.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.2.14 on 2024-08-03 23:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="ToxicDomain",
+            fields=[
+                (
+                    "domain",
+                    models.URLField(
+                        max_length=253,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="domain",
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                (
+                    "is_externally_managed",
+                    models.BooleanField(
+                        default=False,
+                        help_text="True if kept up-to-date using external toxic domain providers, else False",
+                        verbose_name="is externally managed",
+                    ),
+                ),
+            ],
+        ),
+    ]

antispam/models.py

@@ -0,0 +1,19 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+
+class ToxicDomain(models.Model):
+    """Domain marked as spam in public databases"""
+
+    domain = models.URLField(_("domain"), max_length=253, primary_key=True)
+    created = models.DateTimeField(auto_now_add=True)
+    is_externally_managed = models.BooleanField(
+        _("is externally managed"),
+        default=False,
+        help_text=_(
+            "True if kept up-to-date using external toxic domain providers, else False"
+        ),
+    )
+
+    def __str__(self) -> str:
+        return self.domain

core/tests.py

@@ -24,8 +24,10 @@ from django.core.mail import EmailMessage
 from django.test import Client, TestCase
 from django.urls import reverse
 from django.utils.timezone import now
+from model_bakery import baker
 from pytest_django.asserts import assertInHTML, assertRedirects
 
+from antispam.models import ToxicDomain
 from club.models import Membership
 from core.markdown import markdown
 from core.models import AnonymousUser, Group, Page, User
@@ -48,6 +50,10 @@ class TestUserRegistration:
             "captcha_1": "PASSED",
         }
 
+    @pytest.fixture()
+    def scam_domains(self):
+        return [baker.make(ToxicDomain, domain="scammer.spam")]
+
     def test_register_user_form_ok(self, client, valid_payload):
         """Should register a user correctly."""
         assert not User.objects.filter(email=valid_payload["email"]).exists()
@@ -64,14 +70,25 @@ class TestUserRegistration:
                 {"password2": "not the same as password1"},
                 "Les deux mots de passe ne correspondent pas.",
             ),
-            ({"email": "not-an-email"}, "Saisissez une adresse de courriel valide."),
+            (
+                {"email": "not-an-email"},
+                "Saisissez une adresse de courriel valide.",
+            ),
+            (
+                {"email": "not\\an@email.com"},
+                "Saisissez une adresse de courriel valide.",
+            ),
+            (
+                {"email": "legit@scammer.spam"},
+                "Le domaine de l'addresse e-mail n'est pas autorisé.",
+            ),
             ({"first_name": ""}, "Ce champ est obligatoire."),
             ({"last_name": ""}, "Ce champ est obligatoire."),
             ({"captcha_1": "WRONG_CAPTCHA"}, "CAPTCHA invalide"),
         ],
     )
     def test_register_user_form_fail(
-        self, client, valid_payload, payload_edit, expected_error
+        self, client, scam_domains, valid_payload, payload_edit, expected_error
     ):
         """Should not register a user correctly."""
         payload = valid_payload | payload_edit

core/views/forms.py

@@ -16,7 +16,7 @@
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 # Place - Suite 330, Boston, MA 02111-1307, USA.
 #
 #
@@ -45,6 +45,7 @@ from django.utils.translation import gettext_lazy as _
 from phonenumber_field.widgets import RegionalPhoneNumberWidget
 from PIL import Image
 
+from antispam.forms import AntiSpamEmailField
 from core.models import Gift, Page, SithFile, User
 from core.utils import resize_image
 
@@ -194,6 +195,9 @@ class RegisteringForm(UserCreationForm):
     class Meta:
         model = User
         fields = ("first_name", "last_name", "email")
+        field_classes = {
+            "email": AntiSpamEmailField,
+        }
 
 
 class UserProfileForm(forms.ModelForm):

docs/reference/antispam/forms.md

@@ -0,0 +1 @@
+::: antispam.forms

docs/reference/antispam/models.md

@@ -0,0 +1 @@
+::: antispam.models

docs/tutorial/install_advanced.md

@@ -0,0 +1,10 @@
+## Mettre à jour la base de données antispam
+
+L'anti spam nécessite d'être à jour par rapport à des bases de données externe.
+Il existe une commande pour ça qu'il faut lancer régulièrement.
+Lors de la mise en production, il est judicieux de configurer
+un cron pour la mettre à jour au moins une fois par jour.
+
+```bash
+python manage.py update_spam_database
+```

docs/tutorial/structure.md

@@ -64,17 +64,19 @@ sith3/
 │   └── ...
 ├── trombi/ (22)
 │   └── ...
+├── antispam/ (23)
+│   └── ...
 │
-├── .coveragerc (23)
+├── .coveragerc (24)
-├── .envrc (24)
+├── .envrc (25)
 ├── .gitattributes
 ├── .gitignore
 ├── .mailmap
 ├── .env.exemple
-├── manage.py (25)
+├── manage.py (26)
-├── mkdocs.yml (26)
+├── mkdocs.yml (27)
 ├── poetry.lock
-├── pyproject.toml (27)
+├── pyproject.toml (28)
 └── README.md
 ```
 </div>
@@ -112,15 +114,16 @@ sith3/
 19. Application principale du projet, contenant sa configuration.
 20. Gestion des stocks des comptoirs.
 21. Gestion des cotisations des utilisateurs du site.
-22. Gestion des trombinoscopes.
+22. Fonctionalitées pour gérer le spam.
-23. Fichier de configuration de coverage.
+23. Gestion des trombinoscopes.
-24. Fichier de configuration de direnv.
+24. Fichier de configuration de coverage.
-25. Fichier généré automatiquement par Django. C'est lui
+25. Fichier de configuration de direnv.
+26. Fichier généré automatiquement par Django. C'est lui
     qui permet d'appeler des commandes de gestion du projet
     avec la syntaxe `python ./manage.py <nom de la commande>`
-26. Le fichier de configuration de la documentation,
+27. Le fichier de configuration de la documentation,
     avec ses plugins et sa table des matières.
-27. Le fichier où sont déclarés les dépendances et la configuration
+28. Le fichier où sont déclarés les dépendances et la configuration
     de certaines d'entre elles.
     
 

mkdocs.yml

@@ -66,6 +66,7 @@ nav:
     - Gestion des permissions: tutorial/perms.md
     - Gestion des groupes: tutorial/groups.md
     - Etransactions: tutorial/etransaction.md
+    - Installer le projet (Avancé): tutorial/install_advanced.md
   - How-to:
     - L'ORM de Django: howto/querysets.md
     - Gérer les migrations: howto/migrations.md
@@ -80,6 +81,9 @@ nav:
     - accounting:
       - reference/accounting/models.md
       - reference/accounting/views.md
+    - antispam:
+      - reference/antispam/models.md
+      - reference/antispam/forms.md
     - club:
       - reference/club/models.md
       - reference/club/views.md

sith/settings.py

@@ -98,6 +98,7 @@ INSTALLED_APPS = (
     "matmat",
     "pedagogy",
     "galaxy",
+    "antispam",
 )
 
 MIDDLEWARE = (
@@ -204,13 +205,12 @@ SASS_PRECISION = 8
 WSGI_APPLICATION = "sith.wsgi.application"
 
 # Database
-# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
 
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.sqlite3",
         "NAME": BASE_DIR / "db.sqlite3",
-    }
+    },
 }
 
 SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
@@ -673,6 +673,10 @@ SITH_GIFT_LIST = [("AE Tee-shirt", _("AE tee-shirt"))]
 SENTRY_DSN = ""
 SENTRY_ENV = "production"
 
+TOXIC_DOMAINS_PROVIDERS = [
+    "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt",
+]
+
 try:
     from .settings_custom import *