Commit 4fc2a0a6 authored by Samuel GAIST's avatar Samuel GAIST
Browse files

[utils][management] Add command to setup databases

This command will allow to add new databases more easily from
beat.examples. It can either only add new versions or update all.
parent 7b4da722
Pipeline #23453 passed with stage
in 26 minutes and 41 seconds
......@@ -290,7 +290,7 @@ def load_database_folders(filename):
return {}
def upload_database(prefix, name, data):
def upload_database(prefix, name, data, new_only=False):
"""Uploads the database template to the platform
......@@ -344,6 +344,8 @@ def upload_database(prefix, name, data):
else:
logger.info("Added database `%s'", database)
elif new_only:
return True
else: #only updates files
database = database[0]
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
"""
Examples:
To install advanced databases:
$ manage.py update_installed_databases -v1
Note: If you need to specify your own path to the directories containing the
databases, you could just create a simple JSON file as follows::
{
"atnt/1": "/remote/databases/atnt",
"banca/2": "/remote/databases/banca"
}
Then just use the script with the option ``--database-root-file``::
$ manage.py update_installed_databases -v1 --database-root-file=<file.json>
By default, the scripts only installs versions that are not already available.
In order to do a full update of the database use ``--replace``::
$ manage.py update_installed_databases -v1 --replace
It's also possible to do a dry-run to just determine what would be installed
using the option ``--dry-run``::
$ manage.py update_installed_databases -v1 --dry-run
By default, paths to the root of all databases are set to match the Idiap
Research Institute filesystem organisation.
"""
import os
import sys
import logging
import beat.core.database
from django.core.management.base import BaseCommand
from django.conf import settings
from django.contrib.auth.models import User
from beat.web.databases.models import Database
from .install import link_database_versions
from .install import load_database_folders
from .install import upload_database
from .install import list_objects
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'Install the various database up to their latest versions'
def __init__(self):
super(Command, self).__init__()
self.prefix = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))),
'src',
'beat.examples',
)
def add_arguments(self, parser):
from argparse import RawDescriptionHelpFormatter
parser.epilog = __doc__
parser.formatter_class = RawDescriptionHelpFormatter
parser.add_argument('--private', '-p', action='store_true',
dest='private', default=False, help='Set this flag if all ' \
'databases should private')
parser.add_argument('--database-root-file', '-R', type=str,
dest='database_root_file', help='The JSON file containing ' \
'the root directories of the databases installed ' \
'on the platform. If not set or if there is no ' \
'entry in this file for a given database, the root ' \
'directory defined on the JSON database file is used.')
parser.add_argument('--replace', '-r', action='store_true',
dest='replace', default=False, help='Set this flag if all ' \
'databases should be replaced rather than just new added')
parser.add_argument('--dry-run', '-d', action='store_true',
dest='dry_run', default=False, help='Set this flag to ' \
'simulate a run.')
def handle(self, *ignored, **arguments):
# Setup this command's logging level
global logger
arguments['verbosity'] = int(arguments['verbosity'])
if arguments['verbosity'] >= 1:
if arguments['verbosity'] == 1:
logger.setLevel(logging.INFO)
elif arguments['verbosity'] >= 2:
logger.setLevel(logging.DEBUG)
new_only = arguments['replace'] == False
dry_run = arguments['dry_run']
system_user = User.objects.get(username=settings.SYSTEM_ACCOUNT)
user = User.objects.get(username='user')
template_data = dict(
system_user=system_user,
user=user,
private=arguments['private']
)
# Reads database root file, if provided
db_root_file = arguments['database_root_file']
db_root = {}
if db_root_file:
db_root.update(load_database_folders(db_root_file))
source_prefix = os.path.join(self.prefix, 'advanced')
for key in list_objects(self.prefix, 'advanced', 'databases', '*.json'):
template_data.pop('root_folder', None)
if key in db_root:
template_data['root_folder'] = db_root[key]
logger.info('Installing/updating: %s for %s' % (source_prefix, key))
if dry_run:
storage = beat.core.database.Storage(source_prefix, key)
database = Database.objects.filter(name=storage.name,
version=int(storage.version))
if not database:
logger.info("Would create: %s" % key)
elif new_only:
logger.info("Would not do anything: %s" % key)
else:
logger.info("Would update: %s" % key)
else:
success = upload_database(source_prefix, key, template_data, new_only)
if not success:
logger.error("Failed to install %s", key)
if not dry_run:
link_database_versions()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment