diff --git a/beat/backend/python/database.py b/beat/backend/python/database.py
index 6340b7f883af5d5664dbfc7daa1fb623346b069f..2bc552e56242718fe1f44bc1409687ba81245bbf 100755
--- a/beat/backend/python/database.py
+++ b/beat/backend/python/database.py
@@ -382,7 +382,7 @@ class Database(object):
return [k['name'] for k in data]
- def view(self, protocol, name, exc=None):
+ def view(self, protocol, name, exc=None, root_folder=None):
"""Returns the database view, given the protocol and the set name
Parameters:
@@ -426,8 +426,11 @@ class Database(object):
else:
raise #just re-raise the user exception
- return Runner(self._module, self.set(protocol, name), self.prefix,
- self.data['root_folder'], exc)
+ if root_folder is None:
+ root_folder = self.data['root_folder']
+
+ return Runner(self._module, self.set(protocol, name),
+ self.prefix, root_folder, exc)
def json_dumps(self, indent=4):
diff --git a/beat/backend/python/scripts/databases_provider.py b/beat/backend/python/scripts/databases_provider.py
index bdf247e9a8a0246983d38f36582b22c272c1f209..9d5ff461ae533f39eb47cb88da34e9d301e50962 100755
--- a/beat/backend/python/scripts/databases_provider.py
+++ b/beat/backend/python/scripts/databases_provider.py
@@ -150,7 +150,7 @@ def main(arguments=None):
'--disabled-login', '--gecos', '""', '-q',
'beat-nobody'])
if retcode != 0:
- send_error(logger, socket, 'sys', 'Failed to create an user with the UID %s' % args['uid'])
+ message_handler.send_error('Failed to create an user with the UID %s' % args['uid'], 'sys')
return 1
# Next, ensure that the needed files are readable by this user
diff --git a/beat/backend/python/scripts/index.py b/beat/backend/python/scripts/index.py
new file mode 100755
index 0000000000000000000000000000000000000000..28be28d1daea0124fef3c69c4d4d5ec69310b9c1
--- /dev/null
+++ b/beat/backend/python/scripts/index.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.backend.python module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+"""Executes some database views. (%(version)s)
+
+usage:
+ %(prog)s [--debug] [--uid=UID] [--db_root_folder=root_folder] <prefix> <cache> <database> [<protocol> [<set>]]
+ %(prog)s (--help)
+ %(prog)s (--version)
+
+
+arguments:
+ <prefix> Path to the prefix
+ <cache> Path to the cache
+ <database> Full name of the database
+
+
+options:
+ -h, --help Shows this help message and exit
+ -V, --version Shows program's version number and exit
+ -d, --debug Runs in debugging mode
+ --uid=UID UID to run as
+ --db_root_folder=root_folder Root folder to use for the database data (overrides the
+ one declared by the database)
+
+"""
+
+import logging
+
+import os
+import sys
+import docopt
+import pwd
+
+from ..database import Database
+from ..hash import hashDataset
+from ..hash import toPath
+
+
+#----------------------------------------------------------
+
+
+def main(arguments=None):
+
+ # Parse the command-line arguments
+ if arguments is None:
+ arguments = sys.argv[1:]
+
+ package = __name__.rsplit('.', 2)[0]
+ version = package + ' v' + \
+ __import__('pkg_resources').require(package)[0].version
+
+ prog = os.path.basename(sys.argv[0])
+
+ args = docopt.docopt(
+ __doc__ % dict(prog=prog, version=version),
+ argv=arguments,
+ version=version
+ )
+
+
+ # Setup the logging system
+ formatter = logging.Formatter(fmt="[%(asctime)s - index.py - " \
+ "%(name)s] %(levelname)s: %(message)s",
+ datefmt="%d/%b/%Y %H:%M:%S")
+
+ handler = logging.StreamHandler()
+ handler.setFormatter(formatter)
+
+ root_logger = logging.getLogger('beat.backend.python')
+ root_logger.addHandler(handler)
+
+ if args['--debug']:
+ root_logger.setLevel(logging.DEBUG)
+ else:
+ root_logger.setLevel(logging.INFO)
+
+ logger = logging.getLogger(__name__)
+
+
+ if args['--uid']:
+ uid = int(args['--uid'])
+
+ # First create the user (if it doesn't exists)
+ try:
+ user = pwd.getpwuid(uid)
+ except:
+ import subprocess
+ retcode = subprocess.call(['adduser', '--uid', str(uid),
+ '--no-create-home', '--disabled-password',
+ '--disabled-login', '--gecos', '""', '-q',
+ 'beat-nobody'])
+ if retcode != 0:
+ logger.error('Failed to create an user with the UID %d' % uid)
+ return 1
+
+ # Change the current user
+ try:
+ os.setgid(uid)
+ os.setuid(uid)
+ except:
+ import traceback
+ logger.error(traceback.format_exc())
+ return 1
+
+
+ # Check the paths
+ if not os.path.exists(args['<prefix>']):
+ logger.error('Invalid prefix path: %s' % args['<prefix>'])
+ return 1
+
+ if not os.path.exists(args['<cache>']):
+ logger.error('Invalid cache path: %s' % args['<cache>'])
+ return 1
+
+
+ # Indexing
+ try:
+ database = Database(args['<prefix>'], args['<database>'])
+
+ if args['<protocol>'] is None:
+ protocols = database.protocol_names
+ else:
+ protocols = [ args['<protocol>'] ]
+
+ for protocol in protocols:
+
+ if args['<set>'] is None:
+ sets = database.set_names(protocol)
+ else:
+ sets = [ args['<set>'] ]
+
+ for set_name in sets:
+ filename = toPath(hashDataset(args['<database>'], protocol, set_name),
+ suffix='.db')
+
+ view = database.view(protocol, set_name, root_folder=args['--db_root_folder'])
+ view.index(os.path.join(args['<cache>'], filename))
+
+ except Exception as e:
+ import traceback
+ logger.error(traceback.format_exc())
+ return 1
+
+ return 0
+
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/beat/backend/python/test/test_databases_index.py b/beat/backend/python/test/test_databases_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..a31d824b0f99c9762d5d2c905236f3a83ff110b3
--- /dev/null
+++ b/beat/backend/python/test/test_databases_index.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+# #
+# Copyright (c) 2017 Idiap Research Institute, http://www.idiap.ch/ #
+# Contact: beat.support@idiap.ch #
+# #
+# This file is part of the beat.backend.python module of the BEAT platform. #
+# #
+# Commercial License Usage #
+# Licensees holding valid commercial BEAT licenses may use this file in #
+# accordance with the terms contained in a written agreement between you #
+# and Idiap. For further information contact tto@idiap.ch #
+# #
+# Alternatively, this file may be used under the terms of the GNU Affero #
+# Public License version 3 as published by the Free Software and appearing #
+# in the file LICENSE.AGPL included in the packaging of this file. #
+# The BEAT platform is distributed in the hope that it will be useful, but #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
+# or FITNESS FOR A PARTICULAR PURPOSE. #
+# #
+# You should have received a copy of the GNU Affero Public License along #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
+# #
+###############################################################################
+
+
+# Tests for experiment execution
+
+import os
+
+import unittest
+import multiprocessing
+import Queue
+import tempfile
+import shutil
+
+from ..scripts import index
+from ..database import Database
+from ..hash import hashDataset
+from ..hash import toPath
+
+from . import prefix
+from . import tmp_prefix
+
+
+#----------------------------------------------------------
+
+
+class IndexationProcess(multiprocessing.Process):
+
+ def __init__(self, queue, arguments):
+ super(IndexationProcess, self).__init__()
+
+ self.queue = queue
+ self.arguments = arguments
+
+
+ def run(self):
+ self.queue.put('STARTED')
+ index.main(self.arguments)
+
+
+#----------------------------------------------------------
+
+
+class TestDatabaseIndexation(unittest.TestCase):
+
+ def __init__(self, methodName='runTest'):
+ super(TestDatabaseIndexation, self).__init__(methodName)
+ self.databases_indexation_process = None
+ self.working_dir = None
+ self.cache_root = None
+
+
+ def setUp(self):
+ self.shutdown_everything() # In case another test failed badly during its setUp()
+ self.working_dir = tempfile.mkdtemp(prefix=__name__)
+ self.cache_root = tempfile.mkdtemp(prefix=__name__)
+
+
+ def tearDown(self):
+ self.shutdown_everything()
+
+ shutil.rmtree(self.working_dir)
+ shutil.rmtree(self.cache_root)
+
+ self.working_dir = None
+ self.cache_root = None
+ self.data_source = None
+
+
+ def shutdown_everything(self):
+ if self.databases_indexation_process is not None:
+ self.databases_indexation_process.terminate()
+ self.databases_indexation_process.join()
+ del self.databases_indexation_process
+ self.databases_indexation_process = None
+
+
+ def process(self, database, protocol_name=None, set_name=None):
+ args = [
+ prefix.paths[0],
+ self.cache_root,
+ database,
+ ]
+
+ if protocol_name is not None:
+ args.append(protocol_name)
+
+ if set_name is not None:
+ args.append(set_name)
+
+ self.databases_indexation_process = IndexationProcess(multiprocessing.Queue(), args)
+ self.databases_indexation_process.start()
+
+ self.databases_indexation_process.queue.get()
+
+ self.databases_indexation_process.join()
+ del self.databases_indexation_process
+ self.databases_indexation_process = None
+
+
+ def test_one_set(self):
+ self.process('integers_db/1', 'double', 'double')
+
+ expected_files = [
+ hashDataset('integers_db/1', 'double', 'double')
+ ]
+
+ for filename in expected_files:
+ self.assertTrue(os.path.exists(os.path.join(self.cache_root,
+ toPath(filename, suffix='.db'))
+ ))
+
+
+ def test_one_protocol(self):
+ self.process('integers_db/1', 'two_sets')
+
+ expected_files = [
+ hashDataset('integers_db/1', 'two_sets', 'double'),
+ hashDataset('integers_db/1', 'two_sets', 'triple')
+ ]
+
+ for filename in expected_files:
+ self.assertTrue(os.path.exists(os.path.join(self.cache_root,
+ toPath(filename, suffix='.db'))
+ ))
+
+
+ def test_whole_database(self):
+ self.process('integers_db/1')
+
+ expected_files = [
+ hashDataset('integers_db/1', 'double', 'double'),
+ hashDataset('integers_db/1', 'triple', 'triple'),
+ hashDataset('integers_db/1', 'two_sets', 'double'),
+ hashDataset('integers_db/1', 'two_sets', 'triple'),
+ hashDataset('integers_db/1', 'labelled', 'labelled'),
+ hashDataset('integers_db/1', 'different_frequencies', 'double'),
+ ]
+
+ for filename in expected_files:
+ self.assertTrue(os.path.exists(os.path.join(self.cache_root,
+ toPath(filename, suffix='.db'))
+ ))
+
+
+ def test_error(self):
+ self.process('crash/1', 'protocol', 'index_crashes')
+
+ unexpected_files = [
+ hashDataset('crash/1', 'protocol', 'index_crashes'),
+ ]
+
+ for filename in unexpected_files:
+ self.assertFalse(os.path.exists(os.path.join(self.cache_root,
+ toPath(filename, suffix='.db'))
+ ))
diff --git a/beat/backend/python/test/test_databases_provider.py b/beat/backend/python/test/test_databases_provider.py
index dde515a438ff14a6f46dc0666a826b5d08d5a60f..215602f1e5991c4f40e63f4ae54e087007ee8aa4 100644
--- a/beat/backend/python/test/test_databases_provider.py
+++ b/beat/backend/python/test/test_databases_provider.py
@@ -144,10 +144,10 @@ class DatabasesProviderProcess(multiprocessing.Process):
#----------------------------------------------------------
-class TestDatabasesProviderBase(unittest.TestCase):
+class TestDatabasesProvider(unittest.TestCase):
def __init__(self, methodName='runTest'):
- super(TestDatabasesProviderBase, self).__init__(methodName)
+ super(TestDatabasesProvider, self).__init__(methodName)
self.databases_provider_process = None
self.working_dir = None
self.cache_root = None
diff --git a/setup.py b/setup.py
index 5d8585166d4f50a278f0aaf75016670ffdd7f86e..e606a8a8d79dddf9d56fea6397b8ef92b0bd3327 100755
--- a/setup.py
+++ b/setup.py
@@ -69,6 +69,7 @@ setup(
'execute = beat.backend.python.scripts.execute:main',
'describe = beat.backend.python.scripts.describe:main',
'databases_provider = beat.backend.python.scripts.databases_provider:main',
+ 'index = beat.backend.python.scripts.index:main',
],
},