diff --git a/beat/web/databases/api.py b/beat/web/databases/api.py
index a4200815a584f456a0735f69c5e1ac57d88dadc4..e5e05521bcc324dd35549d912135050fe88eb3fc 100755
--- a/beat/web/databases/api.py
+++ b/beat/web/databases/api.py
@@ -25,6 +25,9 @@
# #
###############################################################################
+import os
+import json
+
from django.http import HttpResponse
from django.core.urlresolvers import reverse
@@ -88,11 +91,25 @@ def database_to_json(database, request_user, fields_to_return,
result['hash'] = database.hash
if 'accessibility' in fields_to_return:
- result['accessibility'] = accessibility
+ result['accessibility'] = database.accessibility_for(request_user)
return result
+def clean_paths(declaration):
+ pseudo_path = '/path_to_db_folder'
+ json_data = json.loads(declaration)
+ root_folder = json_data['root_folder']
+ cleaned_folder = os.path.basename(os.path.normpath(root_folder))
+ json_data['root_folder'] = os.path.join(pseudo_path, cleaned_folder)
+ for protocol in json_data['protocols']:
+ for set_ in protocol['sets']:
+ if 'parameters' in set_ and 'annotations' in set_['parameters']:
+ annotations_folder = set_['parameters']['annotations']
+ cleaned_folder = annotations_folder.split('/')[-2:]
+ set_['parameters']['annotations'] = os.path.join(pseudo_path, *cleaned_folder)
+ return json.dumps(json_data)
+
#----------------------------------------------------------
@@ -227,11 +244,12 @@ class RetrieveDatabaseView(views.APIView):
# Retrieve the code
if 'declaration' in fields_to_return:
try:
- result['declaration'] = database.declaration_file.read()
+ declaration = database.declaration_file.read()
except:
logger.error(traceback.format_exc())
return HttpResponse(status=500)
+ result['declaration'] = clean_paths(declaration)
# Retrieve the source code
if 'code' in fields_to_return:
@@ -274,7 +292,7 @@ class RetrieveDatabaseView(views.APIView):
if has_access:
needed_dataformats.append(dataformat)
serializer = ReferencedDataFormatSerializer(needed_dataformats)
- result['referenced_dataformats'] = serializer.data
+ result['needed_dataformats'] = serializer.data
# Return the result
return Response(result)
diff --git a/beat/web/databases/tests.py b/beat/web/databases/tests.py
index 6b530aac46413647e856e7aa22c49e982a2c36e4..45f8f0b3b2ef29a20adfb3d73b5b0884b1b35192 100644
--- a/beat/web/databases/tests.py
+++ b/beat/web/databases/tests.py
@@ -69,14 +69,15 @@ class DatabaseAPIBase(BaseTestCase):
user = User.objects.create_user('jackdoe', 'jackdoe@test.org', '1234')
User.objects.create_user('johndoe', 'johndoe@test.org', '1234')
+ self.db_name = 'test_db'
def tearDown(self):
pass
-class AttestationCreationAPI(DatabaseAPIBase):
+class DatabaseCreationAPI(DatabaseAPIBase):
def setUp(self):
- super(AttestationCreationAPI, self).setUp()
+ super(DatabaseCreationAPI, self).setUp()
self.url = reverse('api_databases:all')
@@ -93,10 +94,9 @@ class AttestationCreationAPI(DatabaseAPIBase):
def test_create_database_failure(self):
self.client.login(username=settings.SYSTEM_ACCOUNT, password='1234')
- db_name = 'test_db'
response = self.client.post(self.url,
json.dumps({
- 'name': db_name,
+ 'name': self.db_name,
'declaration': self.DATABASE
}), content_type='application/json')
@@ -110,16 +110,41 @@ class AttestationCreationAPI(DatabaseAPIBase):
dataformat.share()
self.client.login(username=settings.SYSTEM_ACCOUNT, password='1234')
- db_name = 'test_db'
+
response = self.client.post(self.url,
json.dumps({
- 'name': db_name,
+ 'name': self.db_name,
'declaration': self.DATABASE
}), content_type='application/json')
data = self.checkResponse(response, 201, content_type='application/json')
- self.assertTrue(data['name'] == db_name)
+ self.assertTrue(data['name'] == self.db_name)
databases = Database.objects.all()
self.assertEqual(databases.count(), 1)
+ databases.delete()
+
+
+class DatabaseRetrievalAPI(DatabaseAPIBase):
+
+ def test_retrieve_database(self):
+ (dataformat, errors) = DataFormat.objects.create_dataformat(self.system_user, 'float', '')
+ assert dataformat, errors
+ dataformat.share()
+
+ (database, errors) = Database.objects.create_database(self.db_name, declaration=self.DATABASE)
+ assert database, errors
+ database.share()
+
+ self.client.login(username=settings.SYSTEM_ACCOUNT, password='1234')
+
+ url = reverse('api_databases:object', kwargs={'database_name': self.db_name, 'version': 1})
+
+ response = self.client.get(url, format='json')
+ data = self.checkResponse(response, 200, content_type='application/json')
+
+ declaration = json.loads(data['declaration'])
+ self.assertTrue(declaration['root_folder'].startswith('/path_to_db_folder'))
+
+ database.delete()
\ No newline at end of file
diff --git a/beat/web/utils/management/commands/install.py b/beat/web/utils/management/commands/install.py
index f603713e8464ba4612c5c29d459ea3297a5a0cd1..7c918a5d81cdcbeaf0740edb7ed5e6ca8d18423d 100755
--- a/beat/web/utils/management/commands/install.py
+++ b/beat/web/utils/management/commands/install.py
@@ -290,7 +290,7 @@ def load_database_folders(filename):
return {}
-def upload_database(prefix, name, data):
+def upload_database(prefix, name, data, new_only=False):
"""Uploads the database template to the platform
@@ -344,6 +344,8 @@ def upload_database(prefix, name, data):
else:
logger.info("Added database `%s'", database)
+ elif new_only:
+ return True
else: #only updates files
database = database[0]
diff --git a/beat/web/utils/management/commands/update_installed_databases.py b/beat/web/utils/management/commands/update_installed_databases.py
new file mode 100644
index 0000000000000000000000000000000000000000..21c0417b7d0028a69db9d3ec28e5f9dd09173207
--- /dev/null
+++ b/beat/web/utils/management/commands/update_installed_databases.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.web module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+"""
+Examples:
+
+ To install advanced databases:
+
+ $ manage.py update_installed_databases -v1
+
+ Note: If you need to specify your own path to the directories containing the
+ databases, you could just create a simple JSON file as follows::
+
+ {
+ "atnt/1": "/remote/databases/atnt",
+ "banca/2": "/remote/databases/banca"
+ }
+
+ Then just use the script with the option ``--database-root-file``::
+
+ $ manage.py update_installed_databases -v1 --database-root-file=<file.json>
+
+ By default, the scripts only installs versions that are not already available.
+ In order to do a full update of the database use ``--replace``::
+
+ $ manage.py update_installed_databases -v1 --replace
+
+ It's also possible to do a dry-run to just determine what would be installed
+ using the option ``--dry-run``::
+
+ $ manage.py update_installed_databases -v1 --dry-run
+
+ By default, paths to the root of all databases are set to match the Idiap
+ Research Institute filesystem organisation.
+"""
+
+import os
+import sys
+import logging
+
+import beat.core.database
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User
+
+from beat.web.databases.models import Database
+
+from .install import link_database_versions
+from .install import load_database_folders
+from .install import upload_database
+from .install import list_objects
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+
+ help = 'Install the various database up to their latest versions'
+
+
+ def __init__(self):
+
+ super(Command, self).__init__()
+
+ self.prefix = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))),
+ 'src',
+ 'beat.examples',
+ )
+
+
+ def add_arguments(self, parser):
+
+ from argparse import RawDescriptionHelpFormatter
+ parser.epilog = __doc__
+ parser.formatter_class = RawDescriptionHelpFormatter
+
+ parser.add_argument('--private', '-p', action='store_true',
+ dest='private', default=False, help='Set this flag if all ' \
+ 'databases should private')
+
+ parser.add_argument('--database-root-file', '-R', type=str,
+ dest='database_root_file', help='The JSON file containing ' \
+ 'the root directories of the databases installed ' \
+ 'on the platform. If not set or if there is no ' \
+ 'entry in this file for a given database, the root ' \
+ 'directory defined on the JSON database file is used.')
+
+ parser.add_argument('--replace', '-r', action='store_true',
+ dest='replace', default=False, help='Set this flag if all ' \
+ 'databases should be replaced rather than just new added')
+
+ parser.add_argument('--dry-run', '-d', action='store_true',
+ dest='dry_run', default=False, help='Set this flag to ' \
+ 'simulate a run.')
+
+
+ def handle(self, *ignored, **arguments):
+ # Setup this command's logging level
+ global logger
+ arguments['verbosity'] = int(arguments['verbosity'])
+ if arguments['verbosity'] >= 1:
+ if arguments['verbosity'] == 1:
+ logger.setLevel(logging.INFO)
+ elif arguments['verbosity'] >= 2:
+ logger.setLevel(logging.DEBUG)
+
+ new_only = arguments['replace'] == False
+ dry_run = arguments['dry_run']
+
+ system_user = User.objects.get(username=settings.SYSTEM_ACCOUNT)
+ user = User.objects.get(username='user')
+
+ template_data = dict(
+ system_user=system_user,
+ user=user,
+ private=arguments['private']
+ )
+
+ # Reads database root file, if provided
+ db_root_file = arguments['database_root_file']
+ db_root = {}
+ if db_root_file:
+ db_root.update(load_database_folders(db_root_file))
+
+ source_prefix = os.path.join(self.prefix, 'advanced')
+ for key in list_objects(self.prefix, 'advanced', 'databases', '*.json'):
+ template_data.pop('root_folder', None)
+ if key in db_root:
+ template_data['root_folder'] = db_root[key]
+ logger.info('Installing/updating: %s for %s' % (source_prefix, key))
+ if dry_run:
+ storage = beat.core.database.Storage(source_prefix, key)
+ database = Database.objects.filter(name=storage.name,
+ version=int(storage.version))
+
+ if not database:
+ logger.info("Would create: %s" % key)
+ elif new_only:
+ logger.info("Would not do anything: %s" % key)
+ else:
+ logger.info("Would update: %s" % key)
+ else:
+ success = upload_database(source_prefix, key, template_data, new_only)
+ if not success:
+ logger.error("Failed to install %s", key)
+
+ if not dry_run:
+ link_database_versions()