[utils][management] Add command to setup databases

This command will allow to add new databases more easily from
beat.examples. It can either only add new versions or update all.

beat/web/utils/management/commands/install.py

@@ -290,7 +290,7 @@ def load_database_folders(filename):
     return {}
 
 
-def upload_database(prefix, name, data):
+def upload_database(prefix, name, data, new_only=False):
     """Uploads the database template to the platform
 
 
@@ -344,6 +344,8 @@ def upload_database(prefix, name, data):
         else:
             logger.info("Added database `%s'", database)
 
+    elif new_only:
+        return True
     else: #only updates files
 
         database = database[0]

beat/web/utils/management/commands/update_installed_databases.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+"""
+Examples:
+
+  To install advanced databases:
+
+    $ manage.py update_installed_databases -v1
+
+  Note: If you need to specify your own path to the directories containing the
+  databases, you could just create a simple JSON file as follows::
+
+    {
+      "atnt/1": "/remote/databases/atnt",
+      "banca/2": "/remote/databases/banca"
+    }
+
+  Then just use the script with the option ``--database-root-file``::
+
+      $ manage.py update_installed_databases -v1 --database-root-file=<file.json>
+
+  By default, the scripts only installs versions that are not already available.
+  In order to do a full update of the database use ``--replace``::
+
+      $ manage.py update_installed_databases -v1 --replace
+
+  It's also possible to do a dry-run to just determine what would be installed
+  using the option ``--dry-run``::
+
+      $ manage.py update_installed_databases -v1 --dry-run
+
+  By default, paths to the root of all databases are set to match the Idiap
+  Research Institute filesystem organisation.
+"""
+
+import os
+import sys
+import logging
+
+import beat.core.database
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User
+
+from beat.web.databases.models import Database
+
+from .install import link_database_versions
+from .install import load_database_folders
+from .install import upload_database
+from .install import list_objects
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+
+    help = 'Install the various database up to their latest versions'
+
+
+    def __init__(self):
+
+        super(Command, self).__init__()
+
+        self.prefix = os.path.join(
+            os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))),
+                            'src',
+                            'beat.examples',
+        )
+
+
+    def add_arguments(self, parser):
+
+        from argparse import RawDescriptionHelpFormatter
+        parser.epilog = __doc__
+        parser.formatter_class = RawDescriptionHelpFormatter
+
+        parser.add_argument('--private', '-p', action='store_true',
+                            dest='private', default=False, help='Set this flag if all ' \
+                            'databases should private')
+
+        parser.add_argument('--database-root-file', '-R', type=str,
+                            dest='database_root_file', help='The JSON file containing ' \
+                            'the root directories of the databases installed ' \
+                            'on the platform. If not set or if there is no ' \
+                            'entry in this file for a given database, the root ' \
+                            'directory defined on the JSON database file is used.')
+
+        parser.add_argument('--replace', '-r', action='store_true',
+                            dest='replace', default=False, help='Set this flag if all ' \
+                            'databases should be replaced rather than just new added')
+
+        parser.add_argument('--dry-run', '-d', action='store_true',
+                            dest='dry_run', default=False, help='Set this flag to ' \
+                            'simulate a run.')
+
+
+    def handle(self, *ignored, **arguments):
+        # Setup this command's logging level
+        global logger
+        arguments['verbosity'] = int(arguments['verbosity'])
+        if arguments['verbosity'] >= 1:
+            if arguments['verbosity'] == 1:
+                logger.setLevel(logging.INFO)
+            elif arguments['verbosity'] >= 2:
+                logger.setLevel(logging.DEBUG)
+
+        new_only = arguments['replace'] == False
+        dry_run = arguments['dry_run']
+
+        system_user = User.objects.get(username=settings.SYSTEM_ACCOUNT)
+        user = User.objects.get(username='user')
+
+        template_data = dict(
+            system_user=system_user,
+            user=user,
+            private=arguments['private']
+        )
+
+        # Reads database root file, if provided
+        db_root_file = arguments['database_root_file']
+        db_root = {}
+        if db_root_file:
+            db_root.update(load_database_folders(db_root_file))
+
+        source_prefix = os.path.join(self.prefix, 'advanced')
+        for key in list_objects(self.prefix, 'advanced', 'databases', '*.json'):
+            template_data.pop('root_folder', None)
+            if key in db_root:
+                template_data['root_folder'] = db_root[key]
+            logger.info('Installing/updating: %s for %s' % (source_prefix, key))
+            if dry_run:
+                storage = beat.core.database.Storage(source_prefix, key)
+                database = Database.objects.filter(name=storage.name,
+                                                   version=int(storage.version))
+
+                if not database:
+                    logger.info("Would create: %s" % key)
+                elif new_only:
+                    logger.info("Would not do anything: %s" % key)
+                else:
+                    logger.info("Would update: %s" % key)
+            else:
+                success = upload_database(source_prefix, key, template_data, new_only)
+                if not success:
+                    logger.error("Failed to install %s", key)
+
+        if not dry_run:
+            link_database_versions()