From 96dede5077843fb501fc6da1223c7138de138b5f Mon Sep 17 00:00:00 2001 From: thomas girod <56346771+imperosol@users.noreply.github.com> Date: Tue, 2 May 2023 11:00:23 +0200 Subject: [PATCH] Speed up tests (#638) --- .github/actions/compile_messages/action.yml | 8 ++ .github/actions/setup_project/action.yml | 53 +++++++++++++ .github/actions/setup_xapian/action.yml | 10 +++ .github/workflows/ci.yml | 43 +++++++++++ .github/workflows/unittests.yml | 83 --------------------- accounting/tests.py | 3 - club/tests.py | 3 - com/tests.py | 20 ++--- core/management/commands/setup.py | 8 +- core/tests.py | 47 +++++------- counter/tests.py | 29 +++---- eboutic/tests.py | 1 - election/tests.py | 2 - galaxy/tests.py | 1 - pedagogy/tests.py | 47 +++++------- rootplace/tests.py | 16 ++-- sith/settings.py | 2 + sith/testrunner.py | 9 +++ subscription/tests.py | 6 +- 19 files changed, 191 insertions(+), 200 deletions(-) create mode 100644 .github/actions/compile_messages/action.yml create mode 100644 .github/actions/setup_project/action.yml create mode 100644 .github/actions/setup_xapian/action.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/unittests.yml create mode 100644 sith/testrunner.py diff --git a/.github/actions/compile_messages/action.yml b/.github/actions/compile_messages/action.yml new file mode 100644 index 00000000..aead1e77 --- /dev/null +++ b/.github/actions/compile_messages/action.yml @@ -0,0 +1,8 @@ +name: "Compile messages" +description: "Compile the gettext translation messages" +runs: + using: composite + steps: + - name: Setup project + run: poetry run ./manage.py compilemessages + shell: bash \ No newline at end of file diff --git a/.github/actions/setup_project/action.yml b/.github/actions/setup_project/action.yml new file mode 100644 index 00000000..b1eb6e2a --- /dev/null +++ b/.github/actions/setup_project/action.yml @@ -0,0 +1,53 @@ +name: "Setup project" +description: "Setup Python and Poetry" +runs: + using: composite + steps: + - name: Install apt packages + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: gettext libxapian-dev libgraphviz-dev + version: 1.0 # increment to reset cache + + - name: Install dependencies + run: | + sudo apt update + sudo apt install gettext libxapian-dev libgraphviz-dev + shell: bash + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Load cached Poetry installation + id: cached-poetry + uses: actions/cache@v3 + with: + path: ~/.local + key: poetry-0 # increment to reset cache + + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + shell: bash + run: curl -sSL https://install.python-poetry.org | python3 - + + - name: Check pyproject.toml syntax + shell: bash + run: poetry check + + - name: Load cached dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: poetry install -E testing -E docs + shell: bash + + - name: Compile gettext messages + run: poetry run ./manage.py compilemessages + shell: bash diff --git a/.github/actions/setup_xapian/action.yml b/.github/actions/setup_xapian/action.yml new file mode 100644 index 00000000..6678f581 --- /dev/null +++ b/.github/actions/setup_xapian/action.yml @@ -0,0 +1,10 @@ +name: "Setup xapian" +description: "Setup the xapian indexes" +runs: + using: composite + steps: + - name: Setup xapian index + run: | + mkdir -p /dev/shm/search_indexes + ln -s /dev/shm/search_indexes sith/search_indexes + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..da2bf91f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: Sith 3 CI + +on: + push: + branches: + - master + - taiste + pull_request: + branches: + - master + - taiste + +jobs: + black: + name: Black format + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Setup Project + uses: ./.github/actions/setup_project + - run: poetry run black --check . + + tests: + name: Run tests and generate coverage report + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - uses: ./.github/actions/setup_project + - uses: ./.github/actions/setup_xapian + - uses: ./.github/actions/compile_messages + - name: Run tests + run: poetry run coverage run ./manage.py test + - name: Generate coverage report + run: | + poetry run coverage report + poetry run coverage html + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + with: + name: coverage-report + path: coverage_report diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml deleted file mode 100644 index d4ef94d3..00000000 --- a/.github/workflows/unittests.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Sith3 CI - -on: - pull_request: - branches: [ master, taiste ] - push: - branches: [ master, taiste ] - -jobs: - unittests: - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - # Skip unit testing if no diff on .py files - - name: Check file diff - uses: technote-space/get-diff-action@v6 - id: git-diff - with: - PATTERNS: | - **/*.py - - - name: Set up python - if: steps.git-diff.outputs.diff - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install dependencies - if: steps.git-diff.outputs.diff - run: | - sudo apt-get update - sudo apt-get install gettext libxapian-dev libgraphviz-dev - - - name: Install poetry - if: steps.git-diff.outputs.diff - run: | - python -m pip install --upgrade pip - python -m pip install poetry - - - name: Checking pyproject.toml syntax - if: steps.git-diff.outputs.diff - run: poetry check - - - name: Install project - if: steps.git-diff.outputs.diff - run: poetry install -E testing - - - name: Setup xapian index - if: steps.git-diff.outputs.diff - run: | - mkdir -p /dev/shm/search_indexes - ln -s /dev/shm/search_indexes sith/search_indexes - - - name: Setup project - if: steps.git-diff.outputs.diff - run: poetry run ./manage.py compilemessages - - - name: Launch tests and generate coverage report - if: steps.git-diff.outputs.diff - run: | - poetry run coverage run ./manage.py test - poetry run coverage report - - lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install black - run: | - python -m pip install --upgrade pip - python -m pip install black==23.3.0 - - - name: Check linting - run: black --check . \ No newline at end of file diff --git a/accounting/tests.py b/accounting/tests.py index 4b44e599..6a4ae2b3 100644 --- a/accounting/tests.py +++ b/accounting/tests.py @@ -31,7 +31,6 @@ from accounting.models import ( class RefoundAccountTest(TestCase): def setUp(self): - call_command("populate") self.skia = User.objects.filter(username="skia").first() # reffil skia's account self.skia.customer.amount = 800 @@ -73,7 +72,6 @@ class RefoundAccountTest(TestCase): class JournalTest(TestCase): def setUp(self): - call_command("populate") self.journal = GeneralJournal.objects.filter(id=1).first() def test_permission_granted(self): @@ -101,7 +99,6 @@ class JournalTest(TestCase): class OperationTest(TestCase): def setUp(self): - call_command("populate") self.tomorrow_formatted = (date.today() + timedelta(days=1)).strftime( "%d/%m/%Y" ) diff --git a/club/tests.py b/club/tests.py index e6df319b..30070ba1 100644 --- a/club/tests.py +++ b/club/tests.py @@ -33,7 +33,6 @@ from sith.settings import SITH_BAR_MANAGER class ClubTest(TestCase): def setUp(self): - call_command("populate") self.skia = User.objects.filter(username="skia").first() self.rbatsbak = User.objects.filter(username="rbatsbak").first() self.guy = User.objects.filter(username="guy").first() @@ -379,7 +378,6 @@ class MailingFormTest(TestCase): """Perform validation tests for MailingForm""" def setUp(self): - call_command("populate") self.skia = User.objects.filter(username="skia").first() self.rbatsbak = User.objects.filter(username="rbatsbak").first() self.krophil = User.objects.filter(username="krophil").first() @@ -699,7 +697,6 @@ class ClubSellingViewTest(TestCase): """ def setUp(self): - call_command("populate") self.ae = Club.objects.filter(unix_name="ae").first() def test_page_not_internal_error(self): diff --git a/com/tests.py b/com/tests.py index def450d6..29360dea 100644 --- a/com/tests.py +++ b/com/tests.py @@ -26,9 +26,6 @@ from core.models import User, RealGroup class ComAlertTest(TestCase): - def setUp(self): - call_command("populate") - def test_page_is_working(self): self.client.login(username="comunity", password="plop") response = self.client.get(reverse("com:alert_edit")) @@ -37,9 +34,6 @@ class ComAlertTest(TestCase): class ComInfoTest(TestCase): - def setUp(self): - call_command("populate") - def test_page_is_working(self): self.client.login(username="comunity", password="plop") response = self.client.get(reverse("com:info_edit")) @@ -48,14 +42,16 @@ class ComInfoTest(TestCase): class ComTest(TestCase): - def setUp(self): - call_command("populate") - self.skia = User.objects.filter(username="skia").first() - self.com_group = RealGroup.objects.filter( + @classmethod + def setUpTestData(cls): + cls.skia = User.objects.filter(username="skia").first() + cls.com_group = RealGroup.objects.filter( id=settings.SITH_GROUP_COM_ADMIN_ID ).first() - self.skia.groups.set([self.com_group]) - self.skia.save() + cls.skia.groups.set([cls.com_group]) + cls.skia.save() + + def setUp(self): self.client.login(username=self.skia.username, password="plop") def test_alert_msg(self): diff --git a/core/management/commands/setup.py b/core/management/commands/setup.py index b2e9ae27..cc0ee1ca 100644 --- a/core/management/commands/setup.py +++ b/core/management/commands/setup.py @@ -22,9 +22,6 @@ from django.core.management import call_command class Command(BaseCommand): help = "Set up a new instance of the Sith AE" - def add_arguments(self, parser): - parser.add_argument("--prod", action="store_true") - def handle(self, *args, **options): root_path = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -40,7 +37,4 @@ class Command(BaseCommand): except Exception as e: repr(e) call_command("migrate") - if options["prod"]: - call_command("populate", "--prod") - else: - call_command("populate") + call_command("populate") diff --git a/core/tests.py b/core/tests.py index 7ebe697b..47ee4f8a 100644 --- a/core/tests.py +++ b/core/tests.py @@ -30,11 +30,9 @@ to run these tests : class UserRegistrationTest(TestCase): - def setUp(self): - try: - Group.objects.create(name="root") - except Exception as e: - print(e) + @classmethod + def setUpTestData(cls): + User.objects.all().delete() def test_register_user_form_ok(self): """ @@ -282,19 +280,8 @@ class MarkdownTest(TestCase): class PageHandlingTest(TestCase): def setUp(self): - self.root_group = Group.objects.create(name="root") - u = User( - username="root", - last_name="", - first_name="Bibou", - email="ae.info@utbm.fr", - date_of_birth="1942-06-12", - is_superuser=True, - is_staff=True, - ) - u.set_password("plop") - u.save() self.client.login(username="root", password="plop") + self.root_group = Group.objects.get(name="Root") def test_create_page_ok(self): """ @@ -321,12 +308,20 @@ class PageHandlingTest(TestCase): """ Should create a page correctly """ + # remove all other pages to make sure there is no side effect + Page.objects.all().delete() self.client.post( - reverse("core:page_new"), {"parent": "", "name": "guy", "owner_group": "1"} + reverse("core:page_new"), + {"parent": "", "name": "guy", "owner_group": str(self.root_group.id)}, ) + page = Page.objects.first() response = self.client.post( reverse("core:page_new"), - {"parent": "1", "name": "bibou", "owner_group": "1"}, + { + "parent": str(page.id), + "name": "bibou", + "owner_group": str(self.root_group.id), + }, ) response = self.client.get( reverse("core:page", kwargs={"page_name": "guy/bibou"}) @@ -392,9 +387,6 @@ http://git.an class UserToolsTest(TestCase): - def setUp(self): - call_command("populate") - def test_anonymous_user_unauthorized(self): response = self.client.get(reverse("core:user_tools")) self.assertEqual(response.status_code, 403) @@ -432,13 +424,12 @@ class UserToolsTest(TestCase): class FileHandlingTest(TestCase): + @classmethod + def setUpTestData(cls): + cls.subscriber = User.objects.get(username="subscriber") + def setUp(self): - try: - call_command("populate") - self.subscriber = User.objects.filter(username="subscriber").first() - self.client.login(username="subscriber", password="plop") - except Exception as e: - print(e) + self.client.login(username="subscriber", password="plop") def test_create_folder_home(self): response = self.client.post( diff --git a/counter/tests.py b/counter/tests.py index 13c4403b..ed83e935 100644 --- a/counter/tests.py +++ b/counter/tests.py @@ -30,13 +30,13 @@ from sith.settings import SITH_MAIN_CLUB class CounterTest(TestCase): - def setUp(self): - call_command("populate") - self.skia = User.objects.filter(username="skia").first() - self.sli = User.objects.filter(username="sli").first() - self.krophil = User.objects.filter(username="krophil").first() - self.mde = Counter.objects.filter(name="MDE").first() - self.foyer = Counter.objects.get(id=2) + @classmethod + def setUpTestData(cls): + cls.skia = User.objects.filter(username="skia").first() + cls.sli = User.objects.filter(username="sli").first() + cls.krophil = User.objects.filter(username="krophil").first() + cls.mde = Counter.objects.filter(name="MDE").first() + cls.foyer = Counter.objects.get(id=2) def test_full_click(self): self.client.post( @@ -161,9 +161,7 @@ class CounterTest(TestCase): class CounterStatsTest(TestCase): @classmethod - def setUpClass(cls): - super().setUpClass() - call_command("populate") + def setUpTestData(cls): cls.counter = Counter.objects.filter(id=2).first() cls.krophil = User.objects.get(username="krophil") cls.skia = User.objects.get(username="skia") @@ -171,10 +169,7 @@ class CounterStatsTest(TestCase): cls.root = User.objects.get(username="root") cls.subscriber = User.objects.get(username="subscriber") cls.old_subscriber = User.objects.get(username="old_subscriber") - cls.counter.sellers.add(cls.sli) - cls.counter.sellers.add(cls.root) - cls.counter.sellers.add(cls.skia) - cls.counter.sellers.add(cls.krophil) + cls.counter.sellers.add(cls.sli, cls.root, cls.skia, cls.krophil) barbar = Product.objects.get(code="BARB") @@ -368,7 +363,7 @@ class CounterStatsTest(TestCase): class BillingInfoTest(TestCase): @classmethod - def setUpClass(cls): + def setUpTestData(cls): cls.payload_1 = { "first_name": "Subscribed", "last_name": "User", @@ -385,8 +380,6 @@ class BillingInfoTest(TestCase): "city": "Sète", "country": "FR", } - super().setUpClass() - call_command("populate") def test_edit_infos(self): user = User.objects.get(username="subscriber") @@ -596,7 +589,6 @@ class BillingInfoTest(TestCase): class BarmanConnectionTest(TestCase): def setUp(self): - call_command("populate") self.krophil = User.objects.get(username="krophil") self.skia = User.objects.get(username="skia") self.skia.customer.account = 800 @@ -662,7 +654,6 @@ class StudentCardTest(TestCase): """ def setUp(self): - call_command("populate") self.krophil = User.objects.get(username="krophil") self.sli = User.objects.get(username="sli") diff --git a/eboutic/tests.py b/eboutic/tests.py index b5d4d6ce..af0e5850 100644 --- a/eboutic/tests.py +++ b/eboutic/tests.py @@ -41,7 +41,6 @@ from eboutic.models import Basket class EbouticTest(TestCase): @classmethod def setUpTestData(cls): - call_command("populate") cls.barbar = Product.objects.filter(code="BARB").first() cls.refill = Product.objects.filter(code="15REFILL").first() cls.cotis = Product.objects.filter(code="1SCOTIZ").first() diff --git a/election/tests.py b/election/tests.py index 31934187..03b46481 100644 --- a/election/tests.py +++ b/election/tests.py @@ -9,8 +9,6 @@ from election.models import Election class MainElection(TestCase): def setUp(self): - call_command("populate") - self.election = Election.objects.all().first() self.public_group = Group.objects.get(id=settings.SITH_GROUP_PUBLIC_ID) self.subscriber_group = Group.objects.get(name=settings.SITH_MAIN_MEMBERS_GROUP) diff --git a/galaxy/tests.py b/galaxy/tests.py index d5957a16..c30ec8cf 100644 --- a/galaxy/tests.py +++ b/galaxy/tests.py @@ -31,7 +31,6 @@ from galaxy.models import Galaxy class GalaxyTest(TestCase): def setUp(self): - call_command("populate") self.root = User.objects.get(username="root") self.skia = User.objects.get(username="skia") self.sli = User.objects.get(username="sli") diff --git a/pedagogy/tests.py b/pedagogy/tests.py index 114daba2..baf87545 100644 --- a/pedagogy/tests.py +++ b/pedagogy/tests.py @@ -83,12 +83,12 @@ class UVCreation(TestCase): Test uv creation """ - def setUp(self): - call_command("populate") - self.bibou = User.objects.filter(username="root").first() - self.tutu = User.objects.filter(username="tutu").first() - self.sli = User.objects.filter(username="sli").first() - self.guy = User.objects.filter(username="guy").first() + @classmethod + def setUp(cls): + cls.bibou = User.objects.get(username="root") + cls.tutu = User.objects.get(username="tutu") + cls.sli = User.objects.get(username="sli") + cls.guy = User.objects.get(username="guy") def test_create_uv_admin_success(self): self.client.login(username="root", password="plop") @@ -157,9 +157,6 @@ class UVListTest(TestCase): Test guide display rights """ - def setUp(self): - call_command("populate") - def test_uv_list_display_success(self): # Display for root self.client.login(username="root", password="plop") @@ -192,9 +189,6 @@ class UVDeleteTest(TestCase): Test UV deletion rights """ - def setUp(self): - call_command("populate") - def test_uv_delete_root_success(self): self.client.login(username="root", password="plop") self.client.post( @@ -250,7 +244,6 @@ class UVUpdateTest(TestCase): """ def setUp(self): - call_command("populate") self.bibou = User.objects.filter(username="root").first() self.tutu = User.objects.filter(username="tutu").first() self.uv = UV.objects.get(code="PA00") @@ -341,13 +334,13 @@ class UVCommentCreationAndDisplay(TestCase): Display and creation are the same view """ - def setUp(self): - call_command("populate") - self.bibou = User.objects.filter(username="root").first() - self.tutu = User.objects.filter(username="tutu").first() - self.sli = User.objects.filter(username="sli").first() - self.guy = User.objects.filter(username="guy").first() - self.uv = UV.objects.get(code="PA00") + @classmethod + def setUpTestData(cls): + cls.bibou = User.objects.get(username="root") + cls.tutu = User.objects.get(username="tutu") + cls.sli = User.objects.get(username="sli") + cls.guy = User.objects.get(username="guy") + cls.uv = UV.objects.get(code="PA00") def test_create_uv_comment_admin_success(self): self.client.login(username="root", password="plop") @@ -473,7 +466,6 @@ class UVCommentDeleteTest(TestCase): """ def setUp(self): - call_command("populate") comment_kwargs = create_uv_comment_template( User.objects.get(username="krophil").id ) @@ -533,11 +525,11 @@ class UVCommentUpdateTest(TestCase): Test UVComment update rights """ + @classmethod + def setUpTestData(cls): + cls.krophil = User.objects.get(username="krophil") + def setUp(self): - call_command("populate") - - self.krophil = User.objects.get(username="krophil") - # Prepare a comment comment_kwargs = create_uv_comment_template(self.krophil.id) comment_kwargs["author"] = self.krophil @@ -624,7 +616,6 @@ class UVSearchTest(TestCase): """ def setUp(self): - call_command("populate") call_command("update_index", "pedagogy") def test_get_page_authorized_success(self): @@ -794,8 +785,6 @@ class UVModerationFormTest(TestCase): """ def setUp(self): - call_command("populate") - self.krophil = User.objects.get(username="krophil") # Prepare a comment @@ -1023,8 +1012,6 @@ class UVCommentReportCreateTest(TestCase): """ def setUp(self): - call_command("populate") - self.krophil = User.objects.get(username="krophil") self.tutu = User.objects.get(username="tutu") diff --git a/rootplace/tests.py b/rootplace/tests.py index 3fd31fdb..b6b957db 100644 --- a/rootplace/tests.py +++ b/rootplace/tests.py @@ -18,6 +18,7 @@ from datetime import date, timedelta from django.core.management import call_command from django.test import TestCase from django.urls import reverse +from django.utils.timezone import localtime, now from club.models import Club from core.models import User, RealGroup @@ -27,22 +28,21 @@ from subscription.models import Subscription class MergeUserTest(TestCase): @classmethod - def setUpClass(cls): - super().setUpClass() - call_command("populate") + def setUpTestData(cls): cls.ae = Club.objects.get(unix_name="ae") cls.eboutic = Counter.objects.get(name="Eboutic") cls.barbar = Product.objects.get(code="BARB") cls.barbar.selling_price = 2 cls.barbar.save() cls.root = User.objects.get(username="root") + cls.to_keep = User.objects.create( + username="to_keep", password="plop", email="u.1@utbm.fr" + ) + cls.to_delete = User.objects.create( + username="to_del", password="plop", email="u.2@utbm.fr" + ) def setUp(self) -> None: - super().setUp() - self.to_keep = User(username="to_keep", password="plop", email="u.1@utbm.fr") - self.to_delete = User(username="to_del", password="plop", email="u.2@utbm.fr") - self.to_keep.save() - self.to_delete.save() self.client.login(username="root", password="plop") def test_simple(self): diff --git a/sith/settings.py b/sith/settings.py index 7a74a0fd..2d4a7647 100644 --- a/sith/settings.py +++ b/sith/settings.py @@ -113,6 +113,8 @@ MIDDLEWARE = ( "core.middleware.SignalRequestMiddleware", ) +TEST_RUNNER = "sith.testrunner.SithTestRunner" + ROOT_URLCONF = "sith.urls" TEMPLATES = [ diff --git a/sith/testrunner.py b/sith/testrunner.py new file mode 100644 index 00000000..1112cf89 --- /dev/null +++ b/sith/testrunner.py @@ -0,0 +1,9 @@ +from django.core.management import call_command +from django.test.runner import DiscoverRunner + + +class SithTestRunner(DiscoverRunner): + def setup_databases(self, **kwargs): + res = super().setup_databases(**kwargs) + call_command("populate") + return res diff --git a/subscription/tests.py b/subscription/tests.py index fe0c812d..4e8ec5a1 100644 --- a/subscription/tests.py +++ b/subscription/tests.py @@ -102,9 +102,9 @@ class SubscriptionUnitTest(TestCase): class SubscriptionIntegrationTest(TestCase): - def setUp(self): - call_command("populate") - self.user = User.objects.filter(username="public").first() + @classmethod + def setUp(cls): + cls.user = User.objects.filter(username="public").first() def test_duration_one_month(self): s = Subscription(