[experiments] Add utility and commands to clean-up orphaned caches

beat/web/experiments/management/commands/cleanup_orphaned_caches.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+import logging
+logger = logging.getLogger(__name__)
+
+from django.core.management.base import BaseCommand
+
+from ... import utils
+
+
+class Command(BaseCommand):
+
+    help = 'Sets and resets queue configurations'
+
+
+    def add_arguments(self, parser):
+
+        parser.add_argument('--delete', action='store_true', dest='delete',
+                default=False, help='Really deletes the CachedFiles - ' \
+                        'otherwise only displays what would be deleted')
+
+    def handle(self, *ignored, **arguments):
+
+        # Setup this command's logging level
+        global logger
+        arguments['verbosity'] = int(arguments['verbosity'])
+        if arguments['verbosity'] >= 1:
+            if arguments['verbosity'] == 1: logger.setLevel(logging.INFO)
+            elif arguments['verbosity'] >= 2: logger.setLevel(logging.DEBUG)
+
+        if arguments['delete']:
+            utils.cleanup_orphaned_cachedfiles()
+
+        else:
+            l = utils.list_orphaned_cachedfiles()
+            for c in l: print(c)
+            print('%d CachedFiles are unreacheable' % len(l))

beat/web/experiments/utils.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+'''Utilities for experiment management'''
+
+
+from django.db.models import Count
+
+from .models import CachedFile
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def list_orphaned_cachedfiles():
+    '''Lists orphaned cache files that do not exist in the disk either'''
+
+    q = CachedFile.objects.annotate(Count('blocks')).filter(blocks__count__lt=1)
+    return [c for c in q if not c.exists()]
+
+
+def cleanup_orphaned_cachedfiles():
+    '''Cleans-up orphaned cache files that do not exist in the disk either'''
+
+    for c in list_orphaned_cachedfiles():
+        logger.info("Removing orphaned CachedFile object `%s'..." % c.hash)
+        c.delete()