Merge branch 'skia' into 'master'

core/models: make some tools to repair the SithFiles FS regarding the DB

See merge request ae/Sith!144
This commit is contained in:
Skia 2018-04-18 22:19:17 +02:00
commit 52832eed4d
4 changed files with 172 additions and 20 deletions

View File

@ -0,0 +1,42 @@
# -*- coding:utf-8 -*
#
# Copyright 2018
# - Skia <skia@libskia.so>
#
# Ce fichier fait partie du site de l'Association des Étudiants de l'UTBM,
# http://ae.utbm.fr.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License a published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
#
#
import os
from django.core.management.base import BaseCommand
from django.core.management import call_command
from core.models import SithFile
class Command(BaseCommand):
help = "Recursively check the file system with respect to the DB"
def add_arguments(self, parser):
parser.add_argument('ids', metavar='ID', type=int, nargs='+', help="The file IDs to process")
def handle(self, *args, **options):
root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
files = SithFile.objects.filter(id__in=options['ids']).all()
for f in files:
f._check_fs()

View File

@ -0,0 +1,42 @@
# -*- coding:utf-8 -*
#
# Copyright 2018
# - Skia <skia@libskia.so>
#
# Ce fichier fait partie du site de l'Association des Étudiants de l'UTBM,
# http://ae.utbm.fr.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License a published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
#
#
import os
from django.core.management.base import BaseCommand
from django.core.management import call_command
from core.models import SithFile
class Command(BaseCommand):
help = "Recursively repair the file system with respect to the DB"
def add_arguments(self, parser):
parser.add_argument('ids', metavar='ID', type=int, nargs='+', help="The file IDs to process")
def handle(self, *args, **options):
root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
files = SithFile.objects.filter(id__in=options['ids']).all()
for f in files:
f._repair_fs()

View File

@ -635,11 +635,11 @@ def get_directory(instance, filename):
def get_compressed_directory(instance, filename):
return '.{0}/compressed/{1}'.format(instance.get_parent_path(), filename)
return './.compressed/{0}/{1}'.format(instance.get_parent_path(), filename)
def get_thumbnail_directory(instance, filename):
return '.{0}/thumbnail/{1}'.format(instance.get_parent_path(), filename)
return './.thumbnails/{0}/{1}'.format(instance.get_parent_path(), filename)
class SithFile(models.Model):
@ -759,25 +759,81 @@ class SithFile(models.Model):
self.save()
def move_to(self, parent):
"""Move a file to somewhere else"""
"""
Move a file to a new parent.
`parent` must be a SithFile with the `is_folder=True` property. Otherwise, this function doesn't change
anything.
This is done only at the DB level, so that it's very fast for the user. Indeed, this function doesn't modify
SithFiles recursively, so it stays efficient even with top-level folders.
"""
if not parent.is_folder:
return
import shutil
import os
with transaction.atomic():
if self.is_folder:
old_file_name = self.get_full_path()
else:
old_file_name = self.file.name
self.parent = parent
self.save()
if self.is_folder:
for c in self.children.all():
c.move_to(self)
shutil.rmtree(os.path.join(settings.MEDIA_ROOT, old_file_name))
else:
self.file.save(name=self.name, content=self.file)
os.remove(os.path.join(settings.MEDIA_ROOT, old_file_name))
self.parent = parent
self.clean()
self.save()
def _repair_fs(self):
"""
This function rebuilds recursively the filesystem as it should be
regarding the DB tree.
"""
if self.is_folder:
for c in self.children.all():
c._repair_fs()
return
else:
# First get future parent path and the old file name
# Prepend "." so that we match all relative handling of Django's
# file storage
parent_path = "." + self.parent.get_full_path()
parent_full_path = settings.MEDIA_ROOT + parent_path
print("Parent full path: %s" % parent_full_path)
os.makedirs(parent_full_path, exist_ok=True)
old_path = self.file.name # Should be relative: "./users/skia/bleh.jpg"
new_path = "." + self.get_full_path()
print("Old path: %s " % old_path)
print("New path: %s " % new_path)
try:
# Make this atomic, so that a FS problem rolls back the DB change
with transaction.atomic():
# Set the new filesystem path
self.file.name = new_path
self.save()
print("New file path: %s " % self.file.path)
# Really move at the FS level
if os.path.exists(parent_full_path):
os.rename(settings.MEDIA_ROOT + old_path, settings.MEDIA_ROOT + new_path)
# Empty directories may remain, but that's not really a
# problem, and that can be solved with a simple shell
# command: `find . -type d -empty -delete`
except Exception as e:
print("This file likely had a problem. Here is the exception:")
print(repr(e))
def _check_path_consistence(self):
file_path = str(self.file)
file_full_path = settings.MEDIA_ROOT + file_path
db_path = ".%s" % self.get_full_path()
if not os.path.exists(file_full_path):
print("%s: WARNING: real file does not exists!" % self.id)
print("file path: %s" % file_path, end='')
print(" db path: %s" % db_path)
return False
if file_path != db_path:
print("%s: " % self.id, end='')
print("file path: %s" % file_path, end='')
print(" db path: %s" % db_path)
return False
else:
return True
def _check_fs(self):
if self.is_folder:
for c in self.children.all():
c._check_fs()
return
else:
self._check_path_consistence()
def __getattribute__(self, attr):
if attr == "is_file":

View File

@ -189,6 +189,18 @@ class FileView(CanViewMixin, DetailView, FormMixin):
form_class = AddFilesForm
def handle_clipboard(request, object):
"""
This method handles the clipboard in the view.
This method can fail, since it does not catch the exceptions coming from
below, allowing proper handling in the calling view.
Use this method like this:
FileView.handle_clipboard(request, self.object)
`request` is usually the self.request object in your view
`object` is the SithFile object you want to put in the clipboard, or
where you want to paste the clipboard
"""
if 'delete' in request.POST.keys():
for f_id in request.POST.getlist('file_list'):
sf = SithFile.objects.filter(id=f_id).first()
@ -200,7 +212,6 @@ class FileView(CanViewMixin, DetailView, FormMixin):
for f_id in request.POST.getlist('file_list'):
f_id = int(f_id)
if f_id in [c.id for c in object.children.all()] and f_id not in request.session['clipboard']:
print(f_id)
request.session['clipboard'].append(f_id)
if 'paste' in request.POST.keys():
for f_id in request.session['clipboard']:
@ -221,6 +232,7 @@ class FileView(CanViewMixin, DetailView, FormMixin):
if 'clipboard' not in request.session.keys():
request.session['clipboard'] = []
if request.user.can_edit(self.object):
# XXX this call can fail!
FileView.handle_clipboard(request, self.object)
self.form = self.get_form() # The form handle only the file upload
files = request.FILES.getlist('file_field')