From 36944f1160ae3f020beaa3fe57a6fd05e0d3ede4 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 5 Apr 2016 10:08:28 +0200
Subject: [PATCH] [experiments] Add utility and commands to clean-up orphaned
caches
---
beat/web/experiments/management/__init__.py | 0
.../management/commands/__init__.py | 0
.../commands/cleanup_orphaned_caches.py | 63 +++++++++++++++++++
beat/web/experiments/utils.py | 52 +++++++++++++++
4 files changed, 115 insertions(+)
create mode 100644 beat/web/experiments/management/__init__.py
create mode 100644 beat/web/experiments/management/commands/__init__.py
create mode 100644 beat/web/experiments/management/commands/cleanup_orphaned_caches.py
create mode 100644 beat/web/experiments/utils.py
diff --git a/beat/web/experiments/management/__init__.py b/beat/web/experiments/management/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/beat/web/experiments/management/commands/__init__.py b/beat/web/experiments/management/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/beat/web/experiments/management/commands/cleanup_orphaned_caches.py b/beat/web/experiments/management/commands/cleanup_orphaned_caches.py
new file mode 100644
index 000000000..b52977cf8
--- /dev/null
+++ b/beat/web/experiments/management/commands/cleanup_orphaned_caches.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.web module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+import logging
+logger = logging.getLogger(__name__)
+
+from django.core.management.base import BaseCommand
+
+from ... import utils
+
+
+class Command(BaseCommand):
+
+ help = 'Sets and resets queue configurations'
+
+
+ def add_arguments(self, parser):
+
+ parser.add_argument('--delete', action='store_true', dest='delete',
+ default=False, help='Really deletes the CachedFiles - ' \
+ 'otherwise only displays what would be deleted')
+
+ def handle(self, *ignored, **arguments):
+
+ # Setup this command's logging level
+ global logger
+ arguments['verbosity'] = int(arguments['verbosity'])
+ if arguments['verbosity'] >= 1:
+ if arguments['verbosity'] == 1: logger.setLevel(logging.INFO)
+ elif arguments['verbosity'] >= 2: logger.setLevel(logging.DEBUG)
+
+ if arguments['delete']:
+ utils.cleanup_orphaned_cachedfiles()
+
+ else:
+ l = utils.list_orphaned_cachedfiles()
+ for c in l: print(c)
+ print('%d CachedFiles are unreacheable' % len(l))
diff --git a/beat/web/experiments/utils.py b/beat/web/experiments/utils.py
new file mode 100644
index 000000000..1b25bd01f
--- /dev/null
+++ b/beat/web/experiments/utils.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.web module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+'''Utilities for experiment management'''
+
+
+from django.db.models import Count
+
+from .models import CachedFile
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def list_orphaned_cachedfiles():
+ '''Lists orphaned cache files that do not exist in the disk either'''
+
+ q = CachedFile.objects.annotate(Count('blocks')).filter(blocks__count__lt=1)
+ return [c for c in q if not c.exists()]
+
+
+def cleanup_orphaned_cachedfiles():
+ '''Cleans-up orphaned cache files that do not exist in the disk either'''
+
+ for c in list_orphaned_cachedfiles():
+ logger.info("Removing orphaned CachedFile object `%s'..." % c.hash)
+ c.delete()
--
GitLab