Commit fb3321c3 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Propagated --allow-missing-files to the UBM training

propagating allow missing files

propagating allow missing files

Fixed small bug

Fixed test unit for the allow missing files

Propagated --allow-missing-files to ISV and iVector training and created tests for all combinations

Removed logger

Removed the function is_missing_file

Removed the function is_missing_file
parent e8d4b0b2
Pipeline #13518 passed with stages
in 9 minutes and 55 seconds
...@@ -158,6 +158,7 @@ def execute(args): ...@@ -158,6 +158,7 @@ def execute(args):
algorithm, algorithm,
args.extractor, args.extractor,
args.limit_training_data, args.limit_training_data,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
# train the feature projector # train the feature projector
...@@ -166,6 +167,7 @@ def execute(args): ...@@ -166,6 +167,7 @@ def execute(args):
algorithm, algorithm,
args.extractor, args.extractor,
args.iteration, args.iteration,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
...@@ -183,6 +185,7 @@ def execute(args): ...@@ -183,6 +185,7 @@ def execute(args):
algorithm, algorithm,
args.extractor, args.extractor,
args.limit_training_data, args.limit_training_data,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
# train the feature projector # train the feature projector
...@@ -191,6 +194,7 @@ def execute(args): ...@@ -191,6 +194,7 @@ def execute(args):
algorithm, algorithm,
args.extractor, args.extractor,
args.iteration, args.iteration,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
......
...@@ -83,6 +83,7 @@ def add_isv_jobs(args, job_ids, deps, submitter): ...@@ -83,6 +83,7 @@ def add_isv_jobs(args, job_ids, deps, submitter):
name = 'pro-gmm', name = 'pro-gmm',
number_of_parallel_jobs = args.grid.number_of_projection_jobs, number_of_parallel_jobs = args.grid.number_of_projection_jobs,
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.projection_queue) **args.grid.projection_queue)
deps.append(job_ids['gmm-projection']) deps.append(job_ids['gmm-projection'])
...@@ -90,6 +91,7 @@ def add_isv_jobs(args, job_ids, deps, submitter): ...@@ -90,6 +91,7 @@ def add_isv_jobs(args, job_ids, deps, submitter):
'--sub-task train-isv', '--sub-task train-isv',
name = 'train-isv', name = 'train-isv',
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.training_queue) **args.grid.training_queue)
deps.append(job_ids['isv-training']) deps.append(job_ids['isv-training'])
...@@ -118,12 +120,14 @@ def execute(args): ...@@ -118,12 +120,14 @@ def execute(args):
algorithm, algorithm,
args.extractor, args.extractor,
indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs),
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
# train the feature projector # train the feature projector
elif args.sub_task == 'train-isv': elif args.sub_task == 'train-isv':
tools.train_isv( tools.train_isv(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
else: else:
......
...@@ -96,6 +96,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -96,6 +96,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
name='i-e-%d' % iteration, name='i-e-%d' % iteration,
number_of_parallel_jobs = args.grid.number_of_projection_jobs, number_of_parallel_jobs = args.grid.number_of_projection_jobs,
dependencies = [job_ids['ivector-m-step']] if iteration != args.tv_start_iteration else deps, dependencies = [job_ids['ivector-m-step']] if iteration != args.tv_start_iteration else deps,
allow_missing_files = args.allow_missing_files,
**args.grid.projection_queue) **args.grid.projection_queue)
# M-step # M-step
...@@ -114,6 +115,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -114,6 +115,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
name = 'pro-ivector', name = 'pro-ivector',
number_of_parallel_jobs = args.grid.number_of_projection_jobs, number_of_parallel_jobs = args.grid.number_of_projection_jobs,
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.projection_queue) **args.grid.projection_queue)
deps.append(job_ids['ivector-projection']) deps.append(job_ids['ivector-projection'])
...@@ -123,6 +125,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -123,6 +125,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
'--sub-task train-whitener', '--sub-task train-whitener',
name = 'train-whitener', name = 'train-whitener',
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.training_queue) **args.grid.training_queue)
deps.append(job_ids['whitener-training']) deps.append(job_ids['whitener-training'])
...@@ -133,6 +136,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -133,6 +136,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
name = 'whitened', name = 'whitened',
number_of_parallel_jobs = args.grid.number_of_projection_jobs, number_of_parallel_jobs = args.grid.number_of_projection_jobs,
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.projection_queue) **args.grid.projection_queue)
deps.append(job_ids['whitening-projection']) deps.append(job_ids['whitening-projection'])
...@@ -142,6 +146,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -142,6 +146,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
'--sub-task train-lda', '--sub-task train-lda',
name = 'train-lda', name = 'train-lda',
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.training_queue) **args.grid.training_queue)
deps.append(job_ids['lda-training']) deps.append(job_ids['lda-training'])
...@@ -152,6 +157,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -152,6 +157,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
name = 'lda_projection', name = 'lda_projection',
number_of_parallel_jobs = args.grid.number_of_projection_jobs, number_of_parallel_jobs = args.grid.number_of_projection_jobs,
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.projection_queue) **args.grid.projection_queue)
deps.append(job_ids['lda-projection']) deps.append(job_ids['lda-projection'])
...@@ -161,6 +167,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -161,6 +167,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
'--sub-task train-wccn', '--sub-task train-wccn',
name = 'train-wccn', name = 'train-wccn',
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.training_queue) **args.grid.training_queue)
deps.append(job_ids['wccn-training']) deps.append(job_ids['wccn-training'])
...@@ -171,6 +178,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -171,6 +178,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
name = 'wccn_projection', name = 'wccn_projection',
number_of_parallel_jobs = args.grid.number_of_projection_jobs, number_of_parallel_jobs = args.grid.number_of_projection_jobs,
dependencies = deps, dependencies = deps,
allow_missing_files = args.allow_missing_files,
**args.grid.projection_queue) **args.grid.projection_queue)
deps.append(job_ids['wccn-projection']) deps.append(job_ids['wccn-projection'])
...@@ -179,6 +187,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter): ...@@ -179,6 +187,7 @@ def add_ivector_jobs(args, job_ids, deps, submitter):
job_ids['plda-training'] = submitter.submit( job_ids['plda-training'] = submitter.submit(
'--sub-task train-plda', '--sub-task train-plda',
name = 'train-plda', name = 'train-plda',
allow_missing_files = args.allow_missing_files,
dependencies = deps, dependencies = deps,
**args.grid.training_queue) **args.grid.training_queue)
deps.append(job_ids['plda-training']) deps.append(job_ids['plda-training'])
...@@ -216,12 +225,14 @@ def execute(args): ...@@ -216,12 +225,14 @@ def execute(args):
algorithm, algorithm,
args.extractor, args.extractor,
indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('extracted', 'train_projector'), args.grid.number_of_projection_jobs),
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
elif args.sub_task == 'ivector-e-step': elif args.sub_task == 'ivector-e-step':
tools.ivector_estep( tools.ivector_estep(
algorithm, algorithm,
args.iteration, args.iteration,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
...@@ -237,17 +248,20 @@ def execute(args): ...@@ -237,17 +248,20 @@ def execute(args):
elif args.sub_task == 'ivector-projection': elif args.sub_task == 'ivector-projection':
tools.ivector_project( tools.ivector_project(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
elif args.sub_task == 'train-whitener': elif args.sub_task == 'train-whitener':
tools.train_whitener( tools.train_whitener(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
elif args.sub_task == 'whitening-projection': elif args.sub_task == 'whitening-projection':
tools.whitening_project( tools.whitening_project(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
...@@ -255,12 +269,14 @@ def execute(args): ...@@ -255,12 +269,14 @@ def execute(args):
if algorithm.use_lda: if algorithm.use_lda:
tools.train_lda( tools.train_lda(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
elif args.sub_task == 'lda-projection': elif args.sub_task == 'lda-projection':
if algorithm.use_lda: if algorithm.use_lda:
tools.lda_project( tools.lda_project(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
...@@ -268,12 +284,14 @@ def execute(args): ...@@ -268,12 +284,14 @@ def execute(args):
if algorithm.use_wccn: if algorithm.use_wccn:
tools.train_wccn( tools.train_wccn(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
elif args.sub_task == 'wccn-projection': elif args.sub_task == 'wccn-projection':
if algorithm.use_wccn: if algorithm.use_wccn:
tools.wccn_project( tools.wccn_project(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs), indices = base_tools.indices(fs.training_list('projected_gmm', 'train_projector'), args.grid.number_of_projection_jobs),
force = args.force) force = args.force)
...@@ -281,6 +299,7 @@ def execute(args): ...@@ -281,6 +299,7 @@ def execute(args):
if algorithm.use_plda: if algorithm.use_plda:
tools.train_plda( tools.train_plda(
algorithm, algorithm,
allow_missing_files = args.allow_missing_files,
force = args.force) force = args.force)
elif args.sub_task == 'save-projector': elif args.sub_task == 'save-projector':
......
...@@ -22,7 +22,7 @@ from bob.bio.base.script.verify import main ...@@ -22,7 +22,7 @@ from bob.bio.base.script.verify import main
data_dir = pkg_resources.resource_filename('bob.bio.gmm', 'test/data') data_dir = pkg_resources.resource_filename('bob.bio.gmm', 'test/data')
def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores',''), executable = main): def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores',''), executable = main, allow_missing_files=False):
try: try:
executable(parameters) executable(parameters)
...@@ -52,8 +52,10 @@ def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('sco ...@@ -52,8 +52,10 @@ def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('sco
assert d[0].shape == d[1].shape assert d[0].shape == d[1].shape
# assert that the data order is still correct # assert that the data order is still correct
assert (d[0][:,0:3] == d[1][:, 0:3]).all() assert (d[0][:,0:3] == d[1][:, 0:3]).all()
# assert that the values are OK
assert numpy.allclose(d[0][:,3].astype(float), d[1][:,3].astype(float), 1e-5) if not allow_missing_files:
# assert that the values are OK
assert numpy.allclose(d[0][:,3].astype(float), d[1][:,3].astype(float), 1e-5)
finally: finally:
shutil.rmtree(test_dir) shutil.rmtree(test_dir)
...@@ -77,6 +79,26 @@ def test_gmm_sequential(): ...@@ -77,6 +79,26 @@ def test_gmm_sequential():
_verify(parameters, test_dir, 'test_gmm_sequential', ref_modifier='-gmm') _verify(parameters, test_dir, 'test_gmm_sequential', ref_modifier='-gmm')
def test_gmm_sequential_missingfiles():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.GMM(2, 2, 2)',
'--zt-norm',
'-vs', 'test_gmm_sequential',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_gmm_sequential', ref_modifier='-gmm', allow_missing_files=True)
@bob.bio.base.test.utils.grid_available @bob.bio.base.test.utils.grid_available
def test_gmm_parallel(): def test_gmm_parallel():
from bob.bio.gmm.script.verify_gmm import main from bob.bio.gmm.script.verify_gmm import main
...@@ -100,6 +122,30 @@ def test_gmm_parallel(): ...@@ -100,6 +122,30 @@ def test_gmm_parallel():
_verify(parameters, test_dir, 'test_gmm_parallel', executable=main, ref_modifier='-gmm') _verify(parameters, test_dir, 'test_gmm_parallel', executable=main, ref_modifier='-gmm')
@bob.bio.base.test.utils.grid_available
def test_gmm_parallel_missingfiles():
from bob.bio.gmm.script.verify_gmm import main
test_dir = tempfile.mkdtemp(prefix='bobtest_')
test_database = os.path.join(test_dir, "submitted.sql3")
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.GMM(2, 2, 2)', '--import', 'bob.bio.gmm', 'bob.io.image',
'-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)', '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
'--clean-intermediate',
'--zt-norm',
'-vs', 'test_gmm_parallel',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_gmm_parallel', executable=main, ref_modifier='-gmm', allow_missing_files=True)
def test_isv_sequential(): def test_isv_sequential():
test_dir = tempfile.mkdtemp(prefix='bobtest_') test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters # define dummy parameters
...@@ -118,6 +164,25 @@ def test_isv_sequential(): ...@@ -118,6 +164,25 @@ def test_isv_sequential():
_verify(parameters, test_dir, 'test_isv_sequential', ref_modifier='-isv') _verify(parameters, test_dir, 'test_isv_sequential', ref_modifier='-isv')
def test_isv_sequential_missingfiles():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.ISV(10, number_of_gaussians=2, kmeans_training_iterations=2, gmm_training_iterations=2, isv_training_iterations=2)',
'--zt-norm',
'-vs', 'test_isv_sequential',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_isv_sequential', ref_modifier='-isv', allow_missing_files=True)
@bob.bio.base.test.utils.grid_available @bob.bio.base.test.utils.grid_available
def test_isv_parallel(): def test_isv_parallel():
from bob.bio.gmm.script.verify_isv import main from bob.bio.gmm.script.verify_isv import main
...@@ -141,6 +206,30 @@ def test_isv_parallel(): ...@@ -141,6 +206,30 @@ def test_isv_parallel():
_verify(parameters, test_dir, 'test_isv_parallel', executable=main, ref_modifier='-isv') _verify(parameters, test_dir, 'test_isv_parallel', executable=main, ref_modifier='-isv')
@bob.bio.base.test.utils.grid_available
def test_isv_parallel_missing_files():
from bob.bio.gmm.script.verify_isv import main
test_dir = tempfile.mkdtemp(prefix='bobtest_')
test_database = os.path.join(test_dir, "submitted.sql3")
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.ISV(10, number_of_gaussians=2, kmeans_training_iterations=2, gmm_training_iterations=2, isv_training_iterations=2)', '--import', 'bob.bio.gmm', 'bob.io.image',
'-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)', '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
'--clean-intermediate',
'--zt-norm',
'-vs', 'test_isv_parallel',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_isv_parallel', executable=main, ref_modifier='-isv', allow_missing_files=True)
def test_ivector_cosine_sequential(): def test_ivector_cosine_sequential():
test_dir = tempfile.mkdtemp(prefix='bobtest_') test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters # define dummy parameters
...@@ -159,6 +248,25 @@ def test_ivector_cosine_sequential(): ...@@ -159,6 +248,25 @@ def test_ivector_cosine_sequential():
_verify(parameters, test_dir, 'test_ivector_cosine_sequential', ref_modifier='-ivector-cosine') _verify(parameters, test_dir, 'test_ivector_cosine_sequential', ref_modifier='-ivector-cosine')
def test_ivector_cosine_sequential_missing_files():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.IVector(10, number_of_gaussians=2, kmeans_training_iterations=2, gmm_training_iterations=2, tv_training_iterations=2)',
'--zt-norm',
'-vs', 'test_ivector_cosine_sequential',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_ivector_cosine_sequential', ref_modifier='-ivector-cosine', allow_missing_files=True)
@bob.bio.base.test.utils.grid_available @bob.bio.base.test.utils.grid_available
def test_ivector_cosine_parallel(): def test_ivector_cosine_parallel():
from bob.bio.gmm.script.verify_ivector import main from bob.bio.gmm.script.verify_ivector import main
...@@ -181,6 +289,32 @@ def test_ivector_cosine_parallel(): ...@@ -181,6 +289,32 @@ def test_ivector_cosine_parallel():
_verify(parameters, test_dir, 'test_ivector_cosine_parallel', executable=main, ref_modifier='-ivector-cosine') _verify(parameters, test_dir, 'test_ivector_cosine_parallel', executable=main, ref_modifier='-ivector-cosine')
@bob.bio.base.test.utils.grid_available
def test_ivector_cosine_parallel_missing_files():
from bob.bio.gmm.script.verify_ivector import main
test_dir = tempfile.mkdtemp(prefix='bobtest_')
test_database = os.path.join(test_dir, "submitted.sql3")
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.IVector(10, number_of_gaussians=2, kmeans_training_iterations=2, gmm_training_iterations=2, tv_training_iterations=2)', '--import', 'bob.bio.gmm', 'bob.io.image',
'-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)', '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
'--clean-intermediate',
'--zt-norm',
'-vs', 'test_ivector_cosine_parallel',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_ivector_cosine_parallel', executable=main, ref_modifier='-ivector-cosine', allow_missing_files=True)
def test_ivector_lda_wccn_plda_sequential(): def test_ivector_lda_wccn_plda_sequential():
test_dir = tempfile.mkdtemp(prefix='bobtest_') test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters # define dummy parameters
...@@ -221,6 +355,30 @@ def test_ivector_lda_wccn_plda_parallel(): ...@@ -221,6 +355,30 @@ def test_ivector_lda_wccn_plda_parallel():
_verify(parameters, test_dir, 'test_ivector_lda_wccn_plda_parallel', executable=main, ref_modifier='-ivector-lda-wccn-plda') _verify(parameters, test_dir, 'test_ivector_lda_wccn_plda_parallel', executable=main, ref_modifier='-ivector-lda-wccn-plda')
@bob.bio.base.test.utils.grid_available
def test_ivector_lda_wccn_plda_parallel_missing_files():
from bob.bio.gmm.script.verify_ivector import main
test_dir = tempfile.mkdtemp(prefix='bobtest_')
test_database = os.path.join(test_dir, "submitted.sql3")
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True, probability_of_none=0.5)',
'-e', 'dummy2d',
'-a', 'bob.bio.gmm.algorithm.IVector(10, number_of_gaussians=2, kmeans_training_iterations=2, gmm_training_iterations=2, tv_training_iterations=2, use_lda=True, use_wccn=True, use_plda=True, lda_dim=2, plda_dim_F=2, plda_dim_G=2, plda_training_iterations=2)', '--import', 'bob.bio.gmm', 'bob.io.image',
'-g', 'bob.bio.base.grid.Grid(grid_type = "local", number_of_parallel_processes = 2, scheduler_sleep_time = 0.1)', '-G', test_database, '--run-local-scheduler', '--stop-on-failure',
'--clean-intermediate',
'--zt-norm',
'-vs', 'test_ivector_lda_wccn_plda_parallel',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.gmm',
'--allow-missing-files'
]
_verify(parameters, test_dir, 'test_ivector_lda_wccn_plda_parallel', executable=main, ref_modifier='-ivector-lda-wccn-plda', allow_missing_files=True)
def test_internal_raises(): def test_internal_raises():
test_dir = tempfile.mkdtemp(prefix='bobtest_') test_dir = tempfile.mkdtemp(prefix='bobtest_')
test_database = os.path.join(test_dir, "submitted.sql3") test_database = os.path.join(test_dir, "submitted.sql3")
...@@ -253,3 +411,4 @@ def test_internal_raises(): ...@@ -253,3 +411,4 @@ def test_internal_raises():
nose.tools.assert_raises(ValueError, script, internal) nose.tools.assert_raises(ValueError, script, internal)
shutil.rmtree(test_dir)