Merge pull request #756 from ae-utbm/antispam

Filter blocked emails
This commit is contained in:
thomas girod 2024-08-04 22:36:03 +02:00 committed by GitHub
commit feb6dcbc94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 569 additions and 328 deletions

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
db.sqlite3 *.sqlite3
*.log *.log
*.pyc *.pyc
*.mo *.mo

0
antispam/__init__.py Normal file
View File

10
antispam/admin.py Normal file
View File

@ -0,0 +1,10 @@
from django.contrib import admin
from antispam.models import ToxicDomain
@admin.register(ToxicDomain)
class ToxicDomainAdmin(admin.ModelAdmin):
list_display = ("domain", "is_externally_managed", "created")
search_fields = ("domain", "is_externally_managed", "created")
list_filter = ("is_externally_managed",)

7
antispam/apps.py Normal file
View File

@ -0,0 +1,7 @@
from django.apps import AppConfig
class AntispamConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
verbose_name = "antispam"
name = "antispam"

18
antispam/forms.py Normal file
View File

@ -0,0 +1,18 @@
import re
from django import forms
from django.core.validators import EmailValidator
from django.utils.translation import gettext_lazy as _
from antispam.models import ToxicDomain
class AntiSpamEmailField(forms.EmailField):
"""An email field that email addresses with a known toxic domain."""
def run_validators(self, value: str):
super().run_validators(value)
# Domain part should exist since email validation is guaranteed to run first
domain = re.search(EmailValidator.domain_regex, value)
if ToxicDomain.objects.filter(domain=domain[0]).exists():
raise forms.ValidationError(_("Email domain is not allowed."))

View File

View File

@ -0,0 +1,69 @@
import requests
from django.conf import settings
from django.core.management import BaseCommand
from django.db.models import Max
from django.utils import timezone
from antispam.models import ToxicDomain
class Command(BaseCommand):
"""Update blocked ips/mails database"""
help = "Update blocked ips/mails database"
def add_arguments(self, parser):
parser.add_argument(
"--force", action="store_true", help="Force re-creation even if up to date"
)
def _should_update(self, *, force: bool = False) -> bool:
if force:
return True
oldest = ToxicDomain.objects.filter(is_externally_managed=True).aggregate(
res=Max("created")
)["res"]
return not (oldest and timezone.now() < (oldest + timezone.timedelta(days=1)))
def _download_domains(self, providers: list[str]) -> set[str]:
domains = set()
for provider in providers:
res = requests.get(provider)
if not res.ok:
self.stderr.write(
f"Source {provider} responded with code {res.status_code}"
)
continue
domains |= set(res.content.decode().splitlines())
return domains
def _update_domains(self, domains: set[str]):
# Cleanup database
ToxicDomain.objects.filter(is_externally_managed=True).delete()
# Create database
ToxicDomain.objects.bulk_create(
[
ToxicDomain(domain=domain, is_externally_managed=True)
for domain in domains
],
ignore_conflicts=True,
)
self.stdout.write("Domain database updated")
def handle(self, *args, **options):
if not self._should_update(force=options["force"]):
self.stdout.write("Domain database is up to date")
return
self.stdout.write("Updating domain database")
domains = self._download_domains(settings.TOXIC_DOMAINS_PROVIDERS)
if not domains:
self.stderr.write(
"No domains could be fetched from settings.TOXIC_DOMAINS_PROVIDERS. "
"Please, have a look at your settings."
)
return
self._update_domains(domains)

View File

@ -0,0 +1,35 @@
# Generated by Django 4.2.14 on 2024-08-03 23:05
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="ToxicDomain",
fields=[
(
"domain",
models.URLField(
max_length=253,
primary_key=True,
serialize=False,
verbose_name="domain",
),
),
("created", models.DateTimeField(auto_now_add=True)),
(
"is_externally_managed",
models.BooleanField(
default=False,
help_text="True if kept up-to-date using external toxic domain providers, else False",
verbose_name="is externally managed",
),
),
],
),
]

View File

19
antispam/models.py Normal file
View File

@ -0,0 +1,19 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
class ToxicDomain(models.Model):
"""Domain marked as spam in public databases"""
domain = models.URLField(_("domain"), max_length=253, primary_key=True)
created = models.DateTimeField(auto_now_add=True)
is_externally_managed = models.BooleanField(
_("is externally managed"),
default=False,
help_text=_(
"True if kept up-to-date using external toxic domain providers, else False"
),
)
def __str__(self) -> str:
return self.domain

View File

@ -24,8 +24,10 @@ from django.core.mail import EmailMessage
from django.test import Client, TestCase from django.test import Client, TestCase
from django.urls import reverse from django.urls import reverse
from django.utils.timezone import now from django.utils.timezone import now
from model_bakery import baker
from pytest_django.asserts import assertInHTML, assertRedirects from pytest_django.asserts import assertInHTML, assertRedirects
from antispam.models import ToxicDomain
from club.models import Membership from club.models import Membership
from core.markdown import markdown from core.markdown import markdown
from core.models import AnonymousUser, Group, Page, User from core.models import AnonymousUser, Group, Page, User
@ -48,6 +50,10 @@ class TestUserRegistration:
"captcha_1": "PASSED", "captcha_1": "PASSED",
} }
@pytest.fixture()
def scam_domains(self):
return [baker.make(ToxicDomain, domain="scammer.spam")]
def test_register_user_form_ok(self, client, valid_payload): def test_register_user_form_ok(self, client, valid_payload):
"""Should register a user correctly.""" """Should register a user correctly."""
assert not User.objects.filter(email=valid_payload["email"]).exists() assert not User.objects.filter(email=valid_payload["email"]).exists()
@ -64,14 +70,25 @@ class TestUserRegistration:
{"password2": "not the same as password1"}, {"password2": "not the same as password1"},
"Les deux mots de passe ne correspondent pas.", "Les deux mots de passe ne correspondent pas.",
), ),
({"email": "not-an-email"}, "Saisissez une adresse de courriel valide."), (
{"email": "not-an-email"},
"Saisissez une adresse de courriel valide.",
),
(
{"email": "not\\an@email.com"},
"Saisissez une adresse de courriel valide.",
),
(
{"email": "legit@scammer.spam"},
"Le domaine de l'addresse e-mail n'est pas autorisé.",
),
({"first_name": ""}, "Ce champ est obligatoire."), ({"first_name": ""}, "Ce champ est obligatoire."),
({"last_name": ""}, "Ce champ est obligatoire."), ({"last_name": ""}, "Ce champ est obligatoire."),
({"captcha_1": "WRONG_CAPTCHA"}, "CAPTCHA invalide"), ({"captcha_1": "WRONG_CAPTCHA"}, "CAPTCHA invalide"),
], ],
) )
def test_register_user_form_fail( def test_register_user_form_fail(
self, client, valid_payload, payload_edit, expected_error self, client, scam_domains, valid_payload, payload_edit, expected_error
): ):
"""Should not register a user correctly.""" """Should not register a user correctly."""
payload = valid_payload | payload_edit payload = valid_payload | payload_edit

View File

@ -16,7 +16,7 @@
# details. # details.
# #
# You should have received a copy of the GNU General Public License along with # You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA. # Place - Suite 330, Boston, MA 02111-1307, USA.
# #
# #
@ -45,6 +45,7 @@ from django.utils.translation import gettext_lazy as _
from phonenumber_field.widgets import RegionalPhoneNumberWidget from phonenumber_field.widgets import RegionalPhoneNumberWidget
from PIL import Image from PIL import Image
from antispam.forms import AntiSpamEmailField
from core.models import Gift, Page, SithFile, User from core.models import Gift, Page, SithFile, User
from core.utils import resize_image from core.utils import resize_image
@ -194,6 +195,9 @@ class RegisteringForm(UserCreationForm):
class Meta: class Meta:
model = User model = User
fields = ("first_name", "last_name", "email") fields = ("first_name", "last_name", "email")
field_classes = {
"email": AntiSpamEmailField,
}
class UserProfileForm(forms.ModelForm): class UserProfileForm(forms.ModelForm):

View File

@ -0,0 +1 @@
::: antispam.forms

View File

@ -0,0 +1 @@
::: antispam.models

View File

@ -0,0 +1,10 @@
## Mettre à jour la base de données antispam
L'anti spam nécessite d'être à jour par rapport à des bases de données externe.
Il existe une commande pour ça qu'il faut lancer régulièrement.
Lors de la mise en production, il est judicieux de configurer
un cron pour la mettre à jour au moins une fois par jour.
```bash
python manage.py update_spam_database
```

View File

@ -64,17 +64,19 @@ sith3/
│ └── ... │ └── ...
├── trombi/ (22) ├── trombi/ (22)
│ └── ... │ └── ...
├── antispam/ (23)
│ └── ...
├── .coveragerc (23) ├── .coveragerc (24)
├── .envrc (24) ├── .envrc (25)
├── .gitattributes ├── .gitattributes
├── .gitignore ├── .gitignore
├── .mailmap ├── .mailmap
├── .env.exemple ├── .env.exemple
├── manage.py (25) ├── manage.py (26)
├── mkdocs.yml (26) ├── mkdocs.yml (27)
├── poetry.lock ├── poetry.lock
├── pyproject.toml (27) ├── pyproject.toml (28)
└── README.md └── README.md
``` ```
</div> </div>
@ -112,15 +114,16 @@ sith3/
19. Application principale du projet, contenant sa configuration. 19. Application principale du projet, contenant sa configuration.
20. Gestion des stocks des comptoirs. 20. Gestion des stocks des comptoirs.
21. Gestion des cotisations des utilisateurs du site. 21. Gestion des cotisations des utilisateurs du site.
22. Gestion des trombinoscopes. 22. Fonctionalitées pour gérer le spam.
23. Fichier de configuration de coverage. 23. Gestion des trombinoscopes.
24. Fichier de configuration de direnv. 24. Fichier de configuration de coverage.
25. Fichier généré automatiquement par Django. C'est lui 25. Fichier de configuration de direnv.
26. Fichier généré automatiquement par Django. C'est lui
qui permet d'appeler des commandes de gestion du projet qui permet d'appeler des commandes de gestion du projet
avec la syntaxe `python ./manage.py <nom de la commande>` avec la syntaxe `python ./manage.py <nom de la commande>`
26. Le fichier de configuration de la documentation, 27. Le fichier de configuration de la documentation,
avec ses plugins et sa table des matières. avec ses plugins et sa table des matières.
27. Le fichier où sont déclarés les dépendances et la configuration 28. Le fichier où sont déclarés les dépendances et la configuration
de certaines d'entre elles. de certaines d'entre elles.

File diff suppressed because it is too large Load Diff

View File

@ -66,6 +66,7 @@ nav:
- Gestion des permissions: tutorial/perms.md - Gestion des permissions: tutorial/perms.md
- Gestion des groupes: tutorial/groups.md - Gestion des groupes: tutorial/groups.md
- Etransactions: tutorial/etransaction.md - Etransactions: tutorial/etransaction.md
- Installer le projet (Avancé): tutorial/install_advanced.md
- How-to: - How-to:
- L'ORM de Django: howto/querysets.md - L'ORM de Django: howto/querysets.md
- Gérer les migrations: howto/migrations.md - Gérer les migrations: howto/migrations.md
@ -80,6 +81,9 @@ nav:
- accounting: - accounting:
- reference/accounting/models.md - reference/accounting/models.md
- reference/accounting/views.md - reference/accounting/views.md
- antispam:
- reference/antispam/models.md
- reference/antispam/forms.md
- club: - club:
- reference/club/models.md - reference/club/models.md
- reference/club/views.md - reference/club/views.md

View File

@ -98,6 +98,7 @@ INSTALLED_APPS = (
"matmat", "matmat",
"pedagogy", "pedagogy",
"galaxy", "galaxy",
"antispam",
) )
MIDDLEWARE = ( MIDDLEWARE = (
@ -204,13 +205,12 @@ SASS_PRECISION = 8
WSGI_APPLICATION = "sith.wsgi.application" WSGI_APPLICATION = "sith.wsgi.application"
# Database # Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = { DATABASES = {
"default": { "default": {
"ENGINE": "django.db.backends.sqlite3", "ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3", "NAME": BASE_DIR / "db.sqlite3",
} },
} }
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db" SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
@ -673,6 +673,10 @@ SITH_GIFT_LIST = [("AE Tee-shirt", _("AE tee-shirt"))]
SENTRY_DSN = "" SENTRY_DSN = ""
SENTRY_ENV = "production" SENTRY_ENV = "production"
TOXIC_DOMAINS_PROVIDERS = [
"https://www.stopforumspam.com/downloads/toxic_domains_whole.txt",
]
try: try:
from .settings_custom import * from .settings_custom import *