Merge branch 'skia' into 'master'

core/models: make some tools to repair the SithFiles FS regarding the DB

See merge request ae/Sith!144

core/management/commands/check_fs.py

@@ -0,0 +1,42 @@
+# -*- coding:utf-8 -*
+#
+# Copyright 2018
+# - Skia <skia@libskia.so>
+#
+# Ce fichier fait partie du site de l'Association des Étudiants de l'UTBM,
+# http://ae.utbm.fr.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License a published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
+# Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+#
+
+import os
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+
+from core.models import SithFile
+
+
+class Command(BaseCommand):
+    help = "Recursively check the file system with respect to the DB"
+
+    def add_arguments(self, parser):
+        parser.add_argument('ids', metavar='ID', type=int, nargs='+', help="The file IDs to process")
+
+    def handle(self, *args, **options):
+        root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+        files = SithFile.objects.filter(id__in=options['ids']).all()
+        for f in files:
+            f._check_fs()

core/management/commands/repair_fs.py

@@ -0,0 +1,42 @@
+# -*- coding:utf-8 -*
+#
+# Copyright 2018
+# - Skia <skia@libskia.so>
+#
+# Ce fichier fait partie du site de l'Association des Étudiants de l'UTBM,
+# http://ae.utbm.fr.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License a published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
+# Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+#
+
+import os
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+
+from core.models import SithFile
+
+
+class Command(BaseCommand):
+    help = "Recursively repair the file system with respect to the DB"
+
+    def add_arguments(self, parser):
+        parser.add_argument('ids', metavar='ID', type=int, nargs='+', help="The file IDs to process")
+
+    def handle(self, *args, **options):
+        root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+        files = SithFile.objects.filter(id__in=options['ids']).all()
+        for f in files:
+            f._repair_fs()

core/models.py

@@ -635,11 +635,11 @@ def get_directory(instance, filename):
 
 
 def get_compressed_directory(instance, filename):
-    return '.{0}/compressed/{1}'.format(instance.get_parent_path(), filename)
+    return './.compressed/{0}/{1}'.format(instance.get_parent_path(), filename)
 
 
 def get_thumbnail_directory(instance, filename):
-    return '.{0}/thumbnail/{1}'.format(instance.get_parent_path(), filename)
+    return './.thumbnails/{0}/{1}'.format(instance.get_parent_path(), filename)
 
 
 class SithFile(models.Model):
@@ -759,25 +759,81 @@ class SithFile(models.Model):
             self.save()
 
     def move_to(self, parent):
-        """Move a file to somewhere else"""
+        """
+        Move a file to a new parent.
+        `parent` must be a SithFile with the `is_folder=True` property. Otherwise, this function doesn't change
+        anything.
+        This is done only at the DB level, so that it's very fast for the user. Indeed, this function doesn't modify
+        SithFiles recursively, so it stays efficient even with top-level folders.
+        """
         if not parent.is_folder:
             return
-        import shutil
-        import os
-        with transaction.atomic():
-            if self.is_folder:
-                old_file_name = self.get_full_path()
-            else:
-                old_file_name = self.file.name
         self.parent = parent
+        self.clean()
         self.save()
+
+    def _repair_fs(self):
+        """
+        This function rebuilds recursively the filesystem as it should be
+        regarding the DB tree.
+        """
         if self.is_folder:
             for c in self.children.all():
-                    c.move_to(self)
+                c._repair_fs()
-                shutil.rmtree(os.path.join(settings.MEDIA_ROOT, old_file_name))
+            return
         else:
-                self.file.save(name=self.name, content=self.file)
+            # First get future parent path and the old file name
-                os.remove(os.path.join(settings.MEDIA_ROOT, old_file_name))
+            # Prepend "." so that we match all relative handling of Django's
+            # file storage
+            parent_path = "." + self.parent.get_full_path()
+            parent_full_path = settings.MEDIA_ROOT + parent_path
+            print("Parent full path: %s" % parent_full_path)
+            os.makedirs(parent_full_path, exist_ok=True)
+            old_path = self.file.name # Should be relative: "./users/skia/bleh.jpg"
+            new_path = "." + self.get_full_path()
+            print("Old path: %s " % old_path)
+            print("New path: %s " % new_path)
+            try:
+                # Make this atomic, so that a FS problem rolls back the DB change
+                with transaction.atomic():
+                    # Set the new filesystem path
+                    self.file.name = new_path
+                    self.save()
+                    print("New file path: %s " % self.file.path)
+                    # Really move at the FS level
+                    if os.path.exists(parent_full_path):
+                        os.rename(settings.MEDIA_ROOT + old_path, settings.MEDIA_ROOT + new_path)
+                        # Empty directories may remain, but that's not really a
+                        # problem, and that can be solved with a simple shell
+                        # command: `find . -type d -empty -delete`
+            except Exception as e:
+                print("This file likely had a problem. Here is the exception:")
+                print(repr(e))
+
+    def _check_path_consistence(self):
+        file_path = str(self.file)
+        file_full_path = settings.MEDIA_ROOT + file_path
+        db_path = ".%s" % self.get_full_path()
+        if not os.path.exists(file_full_path):
+            print("%s: WARNING: real file does not exists!" % self.id)
+            print("file path: %s" % file_path, end='')
+            print("  db path: %s" % db_path)
+            return False
+        if file_path != db_path:
+            print("%s: " % self.id, end='')
+            print("file path: %s" % file_path, end='')
+            print("  db path: %s" % db_path)
+            return False
+        else:
+            return True
+
+    def _check_fs(self):
+        if self.is_folder:
+            for c in self.children.all():
+                c._check_fs()
+            return
+        else:
+            self._check_path_consistence()
 
     def __getattribute__(self, attr):
         if attr == "is_file":

core/views/files.py

@@ -189,6 +189,18 @@ class FileView(CanViewMixin, DetailView, FormMixin):
     form_class = AddFilesForm
 
     def handle_clipboard(request, object):
+        """
+        This method handles the clipboard in the view.
+        This method can fail, since it does not catch the exceptions coming from
+        below, allowing proper handling in the calling view.
+        Use this method like this:
+
+            FileView.handle_clipboard(request, self.object)
+
+        `request` is usually the self.request object in your view
+        `object` is the SithFile object you want to put in the clipboard, or
+                 where you want to paste the clipboard
+        """
         if 'delete' in request.POST.keys():
             for f_id in request.POST.getlist('file_list'):
                 sf = SithFile.objects.filter(id=f_id).first()
@@ -200,7 +212,6 @@ class FileView(CanViewMixin, DetailView, FormMixin):
             for f_id in request.POST.getlist('file_list'):
                 f_id = int(f_id)
                 if f_id in [c.id for c in object.children.all()] and f_id not in request.session['clipboard']:
-                    print(f_id)
                     request.session['clipboard'].append(f_id)
         if 'paste' in request.POST.keys():
             for f_id in request.session['clipboard']:
@@ -221,6 +232,7 @@ class FileView(CanViewMixin, DetailView, FormMixin):
         if 'clipboard' not in request.session.keys():
             request.session['clipboard'] = []
         if request.user.can_edit(self.object):
+            # XXX this call can fail!
             FileView.handle_clipboard(request, self.object)
         self.form = self.get_form()  # The form handle only the file upload
         files = request.FILES.getlist('file_field')