Merge branch 'pedagogy_v2_search_api' into 'pedagogy_v2'

Pedagogy v2 search api

See merge request ae/Sith!214

migrate.py

@@ -77,6 +77,7 @@ from forum.models import (
     ForumMessageMeta,
     ForumUserInfo,
 )
+from pedagogy.models import UV, UVComment, UVResult
 
 db = MySQLdb.connect(**settings.OLD_MYSQL_INFOS)
 start = datetime.datetime.now()
@@ -1568,6 +1569,103 @@ def migrate_club_again():
             pass
 
 
+def migrate_pedagogy():
+    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
+
+    print("Migrating UVs")
+    root = User.objects.get(id=0)
+    semester_conversion = {
+        "closed": "CLOSED",
+        "A": "AUTUMN",
+        "P": "SPRING",
+        "AP": "AUTOMN_AND_SPRING",
+    }
+
+    def department_conversion(department):
+        # Default of this enum is HUMA
+        if not department or department == "Humas":
+            return "HUMA"
+        return department
+
+    def convert_number(num, default=0):
+        if not num:
+            return default
+        return num
+
+    def convert_text(text):
+        if not text:
+            return ""
+        return doku_to_markdown(to_unicode(text))
+
+    cur.execute(
+        """
+    SELECT * FROM pedag_uv
+    LEFT JOIN pedag_uv_dept dept
+    ON dept.id_uv = pedag_uv.id_uv
+    """
+    )
+    for uv in cur:
+        UV(
+            id=uv["id_uv"],
+            code=uv["code"],
+            author=root,
+            credit_type=uv["type"],
+            semester=semester_conversion[uv["semestre"]],
+            language="FR",  # No infos in previous guide about that
+            credits=convert_number(uv["guide_credits"]),
+            department=department_conversion(uv["departement"]),
+            title=convert_text(uv["intitule"]),
+            manager=convert_text(uv["responsable"]),
+            objectives=convert_text(uv["guide_objectifs"]),
+            program=convert_text(uv["guide_programme"]),
+            skills="",  # No info in previous guide about that
+            key_concepts="",  # No info either
+            hours_CM=convert_number(uv["guide_c"]),
+            hours_TD=convert_number(uv["guide_td"]),
+            hours_TP=convert_number(uv["guide_tp"]),
+            hours_THE=convert_number(uv["guide_the"]),
+            hours_TE=0,  # No info either
+        ).save()
+
+    print("Migrating UV Comments")
+
+    cur.execute("SELECT * FROM pedag_uv_commentaire")
+
+    for comment in cur:
+        author = User.objects.filter(id=comment["id_utilisateur"]).first()
+        uv = UV.objects.filter(id=comment["id_uv"]).first()
+        if not author or not uv:
+            continue
+        UVComment(
+            id=comment["id_commentaire"],
+            author=author,
+            uv=uv,
+            comment=convert_text(comment["content"]),
+            grade_global=convert_number(comment["note_generale"], -1),
+            grade_utility=convert_number(comment["note_utilite"], -1),
+            grade_interest=convert_number(comment["note_interet"], -1),
+            grade_teaching=convert_number(comment["note_enseignement"], -1),
+            grade_work_load=convert_number(comment["note_travail"], -1),
+            publish_date=comment["date"].replace(tzinfo=timezone("Europe/Paris")),
+        ).save()
+
+    print("Migrating UV Results")
+    cur.execute("SELECT * FROM pedag_resultat")
+
+    for result in cur:
+        author = User.objects.filter(id=comment["id_utilisateur"]).first()
+        uv = UV.objects.filter(id=comment["id_uv"]).first()
+        if not author or not uv:
+            continue
+        UVResult(
+            id=result["id_resultat"],
+            uv=uv,
+            user=author,
+            grade=result["note"],
+            semester=result["semestre"],
+        ).save()
+
+
 def main():
     print("Start at %s" % start)
     # Core
@@ -1590,7 +1688,8 @@ def main():
     # migrate_forum()
     # reset_index('forum')
     # migrate_mailings()
-    migrate_club_again()
+    # migrate_club_again()
+    migrate_pedagogy()
     end = datetime.datetime.now()
     print("End at %s" % end)
     print("Running time: %s" % (end - start))

pedagogy/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.20 on 2019-06-18 08:52
+# Generated by Django 1.11.20 on 2019-06-18 20:07
 from __future__ import unicode_literals
 
 from django.conf import settings
@@ -248,7 +248,7 @@ class Migration(migrations.Migration):
                 ),
                 (
                     "publish_date",
-                    models.DateField(auto_now=True, verbose_name="publish date"),
+                    models.DateTimeField(blank=True, verbose_name="publish date"),
                 ),
                 (
                     "author",

pedagogy/models.py

@@ -24,6 +24,7 @@
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
+from django.utils import timezone
 from django.core import validators
 from django.conf import settings
 
@@ -211,7 +212,12 @@ class UVComment(models.Model):
         null=False,
         default=-1,
     )
-    publish_date = models.DateField(_("publish date"), auto_now=True)
+    publish_date = models.DateTimeField(_("publish date"), blank=True)
+
+    def save(self, *args, **kwargs):
+        if self.publish_date is None:
+            self.publish_date = timezone.now()
+        super(UVComment, self).save(*args, **kwargs)
 
 
 class UVResult(models.Model):

pedagogy/search_indexes.py

@@ -0,0 +1,58 @@
+# -*- coding:utf-8 -*
+#
+# Copyright 2019
+# - Sli <antoine@bartuccio.fr>
+#
+# Ce fichier fait partie du site de l'Association des Étudiants de l'UTBM,
+# http://ae.utbm.fr.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License a published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
+# Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+#
+
+from django.db import models
+
+from haystack import indexes, signals
+
+from core.search_indexes import BigCharFieldIndex
+from pedagogy.models import UV
+
+
+class IndexSignalProcessor(signals.BaseSignalProcessor):
+    """
+    Auto update index on CRUD operations
+    """
+
+    def setup(self):
+        # Listen only to the ``UV`` model.
+        models.signals.post_save.connect(self.handle_save, sender=UV)
+        models.signals.post_delete.connect(self.handle_delete, sender=UV)
+
+    def teardown(self):
+        # Disconnect only to the ``UV`` model.
+        models.signals.post_save.disconnect(self.handle_save, sender=UV)
+        models.signals.post_delete.disconnect(self.handle_delete, sender=UV)
+
+
+class UVIndex(indexes.SearchIndex, indexes.Indexable):
+    """
+    Indexer class for UVs
+    """
+
+    text = BigCharFieldIndex(document=True, use_template=True)
+    auto = indexes.EdgeNgramField(use_template=True)
+
+    def get_model(self):
+        return UV

pedagogy/templates/pedagogy/uv_detail.jinja

@@ -12,6 +12,7 @@
 	<p>{{ object.program|markdown }}</p>
 	<p>{{ object.skills|markdown }}</p>
 	<p>{{ object.key_concepts|markdown }}</p>
+	<p>{% trans %}UV manager: {% endtrans %}{{ object.manager }}</p>
 
 	{% if object.comments.exists() %}
 		<h2>{% trans %}Comments{% endtrans %}</h2>
@@ -21,7 +22,7 @@
 			<p>{{ comment.grade_interest }}</p>
 			<p>{{ comment.grade_teaching }}</p>
 			<p>{{ comment.grade_work_load }}</p>
-			<p>{{ comment.comment }}</p>
+			<p>{{ comment.comment|markdown }}</p>
 			<p>{% trans %}Published: {% endtrans %}{{ comment.publish_date }}</p>
 			<p>{% trans %}Author: {% endtrans %}{{ comment.author }}</p>
 			{% if user.is_owner(comment) %}

pedagogy/templates/search/indexes/pedagogy/uv_auto.txt

@@ -0,0 +1,2 @@
+{{ object.code }}
+{{ object.manager }}

pedagogy/templates/search/indexes/pedagogy/uv_text.txt

@@ -0,0 +1,2 @@
+{{ object.code }}
+{{ object.manager }}

pedagogy/tests.py

@@ -566,3 +566,164 @@ class UVCommentUpdateTest(TestCase):
         )
         self.assertEquals(response.status_code, 200)
         self.assertEquals(self.comment.author, self.krophil)
+
+
+class UVSearchTest(TestCase):
+    """
+    Test UV guide rights for view and API
+    Test that the API is working well
+    """
+
+    def setUp(self):
+        call_command("populate")
+        call_command("update_index", "pedagogy")
+
+    def test_get_page_authorized_success(self):
+        # Test with root user
+        self.client.login(username="root", password="plop")
+        response = self.client.get(reverse("pedagogy:guide"))
+        self.assertEquals(response.status_code, 200)
+
+        # Test with pedagogy admin
+        self.client.login(username="tutu", password="plop")
+        response = self.client.get(reverse("pedagogy:guide"))
+        self.assertEquals(response.status_code, 200)
+
+        # Test with subscribed user
+        self.client.login(username="sli", password="plop")
+        response = self.client.get(reverse("pedagogy:guide"))
+        self.assertEquals(response.status_code, 200)
+
+    def test_get_page_unauthorized_fail(self):
+        # Test with anonymous user
+        response = self.client.get(reverse("pedagogy:guide"))
+        self.assertEquals(response.status_code, 403)
+
+        # Test with not subscribed user
+        self.client.login(username="guy", password="plop")
+        response = self.client.get(reverse("pedagogy:guide"))
+        self.assertEquals(response.status_code, 403)
+
+    def test_search_pa00_success(self):
+        self.client.login(username="sli", password="plop")
+
+        # Search with UV code
+        response = self.client.get(reverse("pedagogy:guide"), {"search": "PA00"})
+        self.assertContains(response, text="PA00")
+
+        # Search with first letter of UV code
+        response = self.client.get(reverse("pedagogy:guide"), {"search": "P"})
+        self.assertContains(response, text="PA00")
+
+        # Search with UV manager
+        response = self.client.get(reverse("pedagogy:guide"), {"search": "HEYBERGER"})
+        self.assertContains(response, text="PA00")
+
+        # Search with department
+        response = self.client.get(reverse("pedagogy:guide"), {"department": "HUMA"})
+        self.assertContains(response, text="PA00")
+
+        # Search with semester
+        response = self.client.get(reverse("pedagogy:guide"), {"semester": "AUTUMN"})
+        self.assertContains(response, text="PA00")
+
+        response = self.client.get(reverse("pedagogy:guide"), {"semester": "SPRING"})
+        self.assertContains(response, text="PA00")
+
+        response = self.client.get(
+            reverse("pedagogy:guide"), {"semester": "AUTOMN_AND_SPRING"}
+        )
+        self.assertContains(response, text="PA00")
+
+        # Search with language
+        response = self.client.get(reverse("pedagogy:guide"), {"language": "FR"})
+        self.assertContains(response, text="PA00")
+
+        # Search with credit type
+        response = self.client.get(reverse("pedagogy:guide"), {"credit_type": "OM"})
+        self.assertContains(response, text="PA00")
+
+        # Search with combinaison of all
+        response = self.client.get(
+            reverse("pedagogy:guide"),
+            {
+                "search": "P",
+                "department": "HUMA",
+                "semester": "AUTUMN",
+                "language": "FR",
+                "credit_type": "OM",
+            },
+        )
+        self.assertContains(response, text="PA00")
+
+        # Test json briefly
+        response = self.client.get(
+            reverse("pedagogy:guide"),
+            {
+                "json": "t",
+                "search": "P",
+                "department": "HUMA",
+                "semester": "AUTUMN",
+                "language": "FR",
+                "credit_type": "OM",
+            },
+        )
+        self.assertJSONEqual(
+            response.content,
+            [
+                {
+                    "model": "pedagogy.uv",
+                    "pk": 1,
+                    "fields": {
+                        "code": "PA00",
+                        "author": 0,
+                        "credit_type": "OM",
+                        "semester": "AUTOMN_AND_SPRING",
+                        "language": "FR",
+                        "credits": 5,
+                        "department": "HUMA",
+                        "title": "Participation dans une association \u00e9tudiante",
+                        "manager": "Laurent HEYBERGER",
+                        "objectives": "* Permettre aux \u00e9tudiants de r\u00e9aliser, pendant un semestre, un projet culturel ou associatif et de le valoriser.",
+                        "program": "* Semestre pr\u00e9c\u00e9dent proposition d'un projet et d'un cahier des charges\n* Evaluation par un jury de six membres\n* Si accord r\u00e9alisation dans le cadre de l'UV\n* Compte-rendu de l'exp\u00e9rience\n* Pr\u00e9sentation",
+                        "skills": "* G\u00e9rer un projet associatif ou une action \u00e9ducative en autonomie:\n* en produisant un cahier des charges qui -d\u00e9finit clairement le contexte du projet personnel -pose les jalons de ce projet -estime de mani\u00e8re r\u00e9aliste les moyens et objectifs du projet -d\u00e9finit exactement les livrables attendus\n    * en \u00e9tant capable de respecter ce cahier des charges ou, le cas \u00e9ch\u00e9ant, de r\u00e9viser le cahier des charges de mani\u00e8re argument\u00e9e.\n* Relater son exp\u00e9rience dans un rapport:\n* qui permettra \u00e0 d'autres \u00e9tudiants de poursuivre les actions engag\u00e9es\n* qui montre la capacit\u00e9 \u00e0 s'auto-\u00e9valuer et \u00e0 adopter une distance critique sur son action.",
+                        "key_concepts": "* Autonomie\n* Responsabilit\u00e9\n* Cahier des charges\n* Gestion de projet",
+                        "hours_CM": 0,
+                        "hours_TD": 0,
+                        "hours_TP": 0,
+                        "hours_THE": 121,
+                        "hours_TE": 4,
+                    },
+                }
+            ],
+        )
+
+    def test_search_pa00_fail(self):
+
+        # Search with UV code
+        response = self.client.get(reverse("pedagogy:guide"), {"search": "IFC"})
+        self.assertNotContains(response, text="PA00")
+
+        # Search with first letter of UV code
+        response = self.client.get(reverse("pedagogy:guide"), {"search": "I"})
+        self.assertNotContains(response, text="PA00")
+
+        # Search with UV manager
+        response = self.client.get(reverse("pedagogy:guide"), {"search": "GILLES"})
+        self.assertNotContains(response, text="PA00")
+
+        # Search with department
+        response = self.client.get(reverse("pedagogy:guide"), {"department": "TC"})
+        self.assertNotContains(response, text="PA00")
+
+        # Search with semester
+        response = self.client.get(reverse("pedagogy:guide"), {"semester": "CLOSED"})
+        self.assertNotContains(response, text="PA00")
+
+        # Search with language
+        response = self.client.get(reverse("pedagogy:guide"), {"language": "EN"})
+        self.assertNotContains(response, text="PA00")
+
+        # Search with credit type
+        response = self.client.get(reverse("pedagogy:guide"), {"credit_type": "TM"})
+        self.assertNotContains(response, text="PA00")

pedagogy/views.py

@@ -31,6 +31,9 @@ from django.views.generic import (
     FormView,
     View,
 )
+from django.core import serializers
+from django.utils import html
+from django.http import HttpResponse
 from django.core.urlresolvers import reverse_lazy
 
 from core.views import (
@@ -41,6 +44,8 @@ from core.views import (
     CanEditPropMixin,
 )
 
+from haystack.query import SearchQuerySet
+
 from pedagogy.forms import UVForm, UVCommentForm
 from pedagogy.models import UV, UVComment
 
@@ -144,6 +149,56 @@ class UVListView(CanViewMixin, CanCreateUVFunctionMixin, ListView):
     ordering = ["code"]
     template_name = "pedagogy/guide.jinja"
 
+    def get(self, *args, **kwargs):
+        resp = super(UVListView, self).get(*args, **kwargs)
+        if not self.request.GET.get("json", None):
+            # Return normal full template response
+            return resp
+
+        # Return serialized response
+        return HttpResponse(
+            serializers.serialize("json", self.get_queryset()),
+            content_type="application/json",
+        )
+
+    def get_queryset(self):
+        queryset = super(UVListView, self).get_queryset()
+        search = self.request.GET.get("search", None)
+
+        additional_filters = {}
+
+        for filter_type in ["credit_type", "language", "department"]:
+            arg = self.request.GET.get(filter_type, None)
+            if arg:
+                additional_filters[filter_type] = arg
+
+        semester = self.request.GET.get("semester", None)
+        if semester:
+            if semester in ["AUTUMN", "SPRING"]:
+                additional_filters["semester__in"] = [semester, "AUTOMN_AND_SPRING"]
+            else:
+                additional_filters["semester"] = semester
+
+        queryset = queryset.filter(**additional_filters)
+        if not search:
+            return queryset
+
+        if len(search) == 1:
+            # It's a search with only one letter
+            # Hastack doesn't work well with only one letter
+            return queryset.filter(code__startswith=search)
+
+        try:
+            qs = (
+                SearchQuerySet()
+                .models(self.model)
+                .autocomplete(auto=html.escape(search))
+            )
+        except TypeError:
+            return self.model.objects.none()
+
+        return queryset.filter(id__in=([o.object.id for o in qs]))
+
 
 class UVCommentReportCreateView(CreateView):
     """