Commit 9831ac7c authored by Samuel GAIST's avatar Samuel GAIST
Browse files

Merge branch 'fix_commands' into '1.4.x'

Fix commands

See merge request !10
parents f2d288ad f3f7785d
Pipeline #17388 passed with stage
......@@ -269,13 +269,12 @@ def execute(prefix, cache, instructions_file):
for name, cfg in configuration['inputs'].items():
cfg['endpoint'] = name
suffix = ''
if cfg.has_key('database'): # Connected to a database output
db = DatabaseStorage(prefix, cfg['database'])
cfg['hash'] = hash.hashDatasetOutput(db.hash(), cfg['protocol'], cfg['set'], cfg['output'])
cfg['path'] = hash.toPath(cfg['hash'], '')
else:
cfg['path'] = hash.toPath(cfg['hash'], '')
cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set'])
suffix = '.db'
cfg['path'] = hash.toPath(cfg['hash'], suffix=suffix)
algo = AlgorithmStorage(prefix, configuration['algorithm'])
......
......@@ -113,12 +113,7 @@ def info(config, paths, sizes, index_start, index_end):
counter = 0
logger.info(' index:')
while f.hasMoreData():
try:
data, start, end = f.next()
except Exception as e:
logger.error("Failed to retrieve the next data: %s", e)
return 1
for data, start, end in f:
size = len(data.pack())
counter += size
......@@ -154,14 +149,7 @@ def view(config, paths, index_start, index_end):
logger.info(' dataformat: %s', f.dataformat.name)
while f.hasMoreData():
try:
data, start, end = f.next()
except Exception as e:
logger.error("Failed to retrieve the next data: %s", e)
return 1
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
......
This diff is collapsed.
This diff is collapsed.
......@@ -28,10 +28,15 @@
# Basic setup for command test
import os
import sys
import tempfile
import shutil
import subprocess
import pkg_resources
import six.moves.urllib as urllib
from beat.core.test import prefix, tmp_prefix, teardown_package
from beat.core.test import tmp_prefix, teardown_package
platform = os.environ.get('BEAT_CMDLINE_TEST_PLATFORM', '')
......@@ -65,3 +70,28 @@ else:
user = 'user'
token = '4'
if sys.platform == 'darwin':
prefix_folder = tempfile.mkdtemp(prefix=__name__,
suffix='.prefix',
dir='/tmp')
else:
prefix_folder = tempfile.mkdtemp(prefix=__name__,
suffix='.prefix')
prefix = os.path.join(prefix_folder, 'prefix')
def setup_package():
prefixes = [
pkg_resources.resource_filename('beat.backend.python.test', 'prefix'),
pkg_resources.resource_filename('beat.core.test', 'prefix')
]
for path in prefixes:
subprocess.check_call(['rsync', '-arz', path, prefix_folder])
def teardown_package():
shutil.rmtree(prefix_folder)
{
"algorithm": "user/integers_echo/1",
"algorithm": "legacy/echo/1",
"channel": "main",
"inputs": {
"in_data": {
"in": {
"hash": "abcdef0123456789AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"channel": "main"
}
},
"outputs": {
"out_data": {
"out": {
"channel": "main"
}
},
......@@ -16,6 +16,6 @@
},
"environment": {
"name": "Python 2.7",
"version": "1.2.0"
"version": "1.3.0"
}
}
{
"algorithm": "user/integers_echo/1",
"algorithm": "legacy/echo/1",
"channel": "main",
"inputs": {
"in_data": {
"in": {
"database": "integers_db/1",
"protocol": "double",
"set": "double",
......@@ -11,7 +11,7 @@
}
},
"outputs": {
"out_data": {
"out": {
"channel": "main"
}
},
......@@ -19,6 +19,6 @@
},
"environment": {
"name": "Python 2.7",
"version": "1.2.0"
"version": "1.3.0"
}
}
{
"algorithm": "user/sum/1",
"algorithm": "legacy/add/1",
"channel": "main",
"inputs": {
"a": {
"in1": {
"database": "integers_db/1",
"protocol": "double",
"set": "double",
"output": "a",
"channel": "main"
},
"b": {
"in2": {
"hash": "abcdef0123456789AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"channel": "main"
}
},
"outputs": {
"sum": {
"out": {
"channel": "main"
}
},
......@@ -23,6 +23,6 @@
},
"environment": {
"name": "Python 2.7",
"version": "1.2.0"
"version": "1.3.0"
}
}
......@@ -11,6 +11,6 @@
},
"environment": {
"name": "Python 2.7",
"version": "1.2.0"
"version": "1.3.0"
}
}
......@@ -31,13 +31,27 @@
import nose.tools
import os
import shutil
import json
from . import platform, disconnected, prefix, tmp_prefix, user, token
from ..common import Selector
from ..scripts.beat import main
from beat.core.test.utils import slow, cleanup, skipif
from beat.core.algorithm import Storage
from beat.core.database import Database
from beat.backend.python.hash import hashDataset
from beat.backend.python.hash import toPath
def index_db_from_instructions(input_field):
database = Database(prefix, input_field['database'])
view = database.view(input_field['protocol'], input_field['set'])
filename = toPath(hashDataset(input_field['database'],
input_field['protocol'],
input_field['set']),
suffix='.db')
view.index(os.path.join(tmp_prefix, filename))
def call(*args, **kwargs):
'''A central mechanism to call the main routine with the right parameters'''
......@@ -73,7 +87,7 @@ def test_local_list():
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_pull_one():
obj = 'user/integers_add/1'
obj = 'legacy/integers_add/1'
nose.tools.eq_(call('pull', obj, prefix=tmp_prefix), 0)
s = Storage(tmp_prefix, obj)
assert s.exists()
......@@ -90,7 +104,7 @@ def test_pull_all():
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_diff():
obj = 'user/integers_add/1'
obj = 'legacy/integers_add/1'
nose.tools.eq_(call('pull', obj, prefix=tmp_prefix), 0)
# quickly modify the user algorithm by emptying it
......@@ -110,18 +124,18 @@ def test_status():
def test_check_valid():
obj = 'user/valid_algorithm/1'
obj = 'legacy/valid_algorithm/1'
nose.tools.eq_(call('check', obj), 0)
def test_check_invalid():
obj = 'user/no_inputs_declarations/1'
obj = 'legacy/no_inputs_declarations/1'
nose.tools.eq_(call('check', obj), 1)
@nose.tools.with_setup(teardown=cleanup)
def test_create(obj=None):
obj = obj or 'user/algorithm/1'
obj = obj or 'legacy/algorithm/1'
nose.tools.eq_(call('create', obj, prefix=tmp_prefix), 0)
s = Storage(tmp_prefix, obj)
assert s.exists()
......@@ -130,9 +144,9 @@ def test_create(obj=None):
@nose.tools.with_setup(teardown=cleanup)
def test_new_version():
obj = 'user/algorithm/1'
obj = 'legacy/algorithm/1'
test_create(obj)
obj2 = 'user/algorithm/2'
obj2 = 'legacy/algorithm/2'
nose.tools.eq_(call('version', obj, prefix=tmp_prefix), 0)
s = Storage(tmp_prefix, obj2)
assert s.exists()
......@@ -144,9 +158,9 @@ def test_new_version():
@nose.tools.with_setup(teardown=cleanup)
def test_fork():
obj = 'user/algorithm/1'
obj = 'legacy/algorithm/1'
test_create(obj)
obj2 = 'user/different/1'
obj2 = 'legacy/different/1'
nose.tools.eq_(call('fork', obj, obj2, prefix=tmp_prefix), 0)
s = Storage(tmp_prefix, obj2)
assert s.exists()
......@@ -158,7 +172,7 @@ def test_fork():
@nose.tools.with_setup(teardown=cleanup)
def test_delete_local():
obj = 'user/algorithm/1'
obj = 'legacy/algorithm/1'
storage = test_create(obj)
# quickly make sure it exists
......@@ -172,7 +186,7 @@ def test_delete_local():
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_push_and_delete():
obj = 'user/newobject/1'
obj = 'legacy/newobject/1'
test_create(obj)
# now push the new object and then delete it remotely
......@@ -184,6 +198,11 @@ def test_push_and_delete():
@nose.tools.with_setup(teardown=cleanup)
def test_execute_algorithm_using_database():
instructions = os.path.join(os.path.dirname(__file__), 'instructions/algo_using_database.json')
with open(instructions) as instruction_file:
instructions_data = json.load(instruction_file)
input_field = instructions_data['inputs']['in']
index_db_from_instructions(input_field)
nose.tools.eq_(call('execute', instructions, cache=tmp_prefix), 0)
......@@ -218,6 +237,11 @@ def test_execute_algorithm_using_database_and_cached_files():
instructions = os.path.join(instructions_dir, 'algo_using_database_and_cached_files.json')
with open(instructions) as instruction_file:
instructions_data = json.load(instruction_file)
input_field = instructions_data['inputs']['in1']
index_db_from_instructions(input_field)
nose.tools.eq_(call('execute', instructions, cache=tmp_prefix), 0)
......
......@@ -33,7 +33,10 @@ import os
import nose.tools
from . import prefix, tmp_prefix
from .utils import index_experiment_dbs
from ..scripts.beat import main
from beat.core.test.utils import cleanup, slow
......@@ -50,7 +53,11 @@ def call(*args, **kwargs):
def setup_module():
call('experiments', 'run', 'user/user/double_triangle/1/double_triangle')
experiment_name = 'user/user/double_triangle/1/double_triangle'
index_experiment_dbs(experiment_name)
call('experiments', 'run', experiment_name)
def teardown_module():
cleanup()
......
......@@ -31,11 +31,16 @@
import nose.tools
from . import platform, disconnected, prefix, tmp_prefix, user, token
from ..scripts.beat import main
from beat.core.test.utils import slow, cleanup, skipif
from beat.core.database import Storage, Database
def index_integer_db():
call('index', 'integers_db/1')
def call(*args, **kwargs):
'''A central mechanism to call the main routine with the right parameters'''
......@@ -106,22 +111,28 @@ def test_status():
nose.tools.eq_(call('status', prefix=tmp_prefix), 0)
@nose.tools.with_setup(setup=index_integer_db, teardown=cleanup)
def test_view_good():
nose.tools.eq_(call('view', 'integers_db/1/double/double'), 0)
@nose.tools.with_setup(setup=index_integer_db, teardown=cleanup)
def test_view_unknown_protocol():
nose.tools.eq_(call('view', 'integers_db/1/single/double'), 1)
@nose.tools.with_setup(setup=index_integer_db, teardown=cleanup)
def test_view_unknown_set():
nose.tools.eq_(call('view', 'integers_db/1/double/single'), 1)
@nose.tools.with_setup(setup=index_integer_db, teardown=cleanup)
def test_view_bad():
nose.tools.eq_(call('view', 'integers_db/1/two_sets'), 1)
@nose.tools.with_setup(setup=index_integer_db, teardown=cleanup)
def test_view_invalid():
nose.tools.eq_(call('view', 'invalid/1/default/set'), 1)
......@@ -141,12 +152,6 @@ def test_list_index_good():
nose.tools.eq_(call('index', '--list', 'integers_db/1'), 0)
@nose.tools.with_setup(teardown=cleanup)
def test_checksum_index_good():
nose.tools.eq_(call('index', 'integers_db/1'), 0)
nose.tools.eq_(call('index', '--checksum', 'integers_db/1'), 0)
@nose.tools.with_setup(teardown=cleanup)
def test_delete_index_good():
nose.tools.eq_(call('index', 'integers_db/1'), 0)
......@@ -155,7 +160,7 @@ def test_delete_index_good():
@nose.tools.with_setup(teardown=cleanup)
def test_index_all(): #bad and good, return != 0
expected_errors = 19
expected_errors = 16
existing_errors = call('index')
assert existing_errors >= expected_errors, "There should be at least %d " \
"errors on installed databases, but I've found only %d" % \
......
......@@ -33,10 +33,16 @@ import sys
import nose.tools
from . import platform, disconnected, prefix, tmp_prefix, user, token
from .utils import index_experiment_dbs
from ..common import Selector
from ..scripts.beat import main
from beat.core.test.utils import slow, cleanup, skipif
from beat.core.experiment import Storage, Experiment
from beat.core.experiment import Storage
def setup_experiments():
index_experiment_dbs('user/user/double_triangle/1/double_triangle')
index_experiment_dbs('user/user/integers_addition/1/integers_addition')
def call(*args, **kwargs):
......@@ -144,14 +150,15 @@ def test_delete_local():
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_run_integers_addition_1():
obj = 'user/user/integers_addition/1/integers_addition'
# index_db_for_exp(obj)
nose.tools.eq_(call('run', obj, cache=tmp_prefix), 0)
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_list_integers_addition_1_cache():
obj = 'user/user/integers_addition/1/integers_addition'
nose.tools.eq_(call('run', obj, cache=tmp_prefix), 0)
......@@ -159,7 +166,7 @@ def test_list_integers_addition_1_cache():
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_checksum_integers_addition_1_cache():
obj = 'user/user/integers_addition/1/integers_addition'
nose.tools.eq_(call('run', obj, cache=tmp_prefix), 0)
......@@ -167,7 +174,7 @@ def test_checksum_integers_addition_1_cache():
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_delete_integers_addition_1_cache():
obj = 'user/user/integers_addition/1/integers_addition'
nose.tools.eq_(call('run', obj, cache=tmp_prefix), 0)
......@@ -175,14 +182,14 @@ def test_delete_integers_addition_1_cache():
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_run_double_triangle_1():
obj = 'user/user/double_triangle/1/double_triangle'
nose.tools.eq_(call('run', obj, cache=tmp_prefix), 0)
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_run_single_error_1_local():
# When running locally, the module with the error is loaded
# inside the currently running process and will return '1'.
......@@ -200,7 +207,7 @@ def test_run_single_error_1_docker():
@slow
@nose.tools.with_setup(teardown=cleanup)
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_run_single_error_twice_local():
# This one makes sure our output reset is working properly. Both tries should
# give out the same error.
......
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.cmdline module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
"""
Helper functions to run tests
"""
import os
from beat.core.experiment import Experiment
from beat.core.hash import toPath
from beat.core.hash import hashDataset
from . import prefix, tmp_prefix
def index_experiment_dbs(experiment_name):
"""
Index all databases for given experiment
"""
experiment = Experiment(prefix, experiment_name)
assert experiment.valid, '\n * %s' % '\n * '.join(experiment.errors)
for block_name, infos in experiment.datasets.items():
view = infos['database'].view(infos['protocol'], infos['set'])
filename = toPath(hashDataset(infos['database'].name,
infos['protocol'],
infos['set']),
suffix='.db')
view.index(os.path.join(tmp_prefix, filename))
......@@ -14,8 +14,8 @@ eggs = beat.cmdline
ipdb
[sources]
beat.core = git https://gitlab.idiap.ch/beat/beat.core
beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python
beat.core = git https://gitlab.idiap.ch/beat/beat.core branch=1.6.x
beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python branch=1.5.x
[scripts]
recipe = bob.buildout:scripts
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment