Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.core
Commits
9af90457
Commit
9af90457
authored
Jun 26, 2020
by
Amir MOHAMMADI
Browse files
Merge branch '64_add_optionnal_environment_entry_to_database' into 'master'
Add optionnal environment entry to database See merge request
!122
parents
72737fdd
7b28f3aa
Pipeline
#40697
passed with stages
in 17 minutes and 59 seconds
Changes
34
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
beat/core/dock.py
View file @
9af90457
...
...
@@ -57,6 +57,8 @@ from packaging import version
from
beat.core
import
stats
from
.utils
import
build_env_name
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -91,9 +93,10 @@ class Host(object):
)
=
self
.
_discover_environments_using_labels
()
if
not
self
.
db_environments
and
not
self
.
processing_environments
:
self
.
processing_environments
,
self
.
db_environments
=
(
self
.
_discover_environments_using_describe
()
)
(
self
.
processing_environments
,
self
.
db_environments
,
)
=
self
.
_discover_environments_using_describe
()
# (If necessary) Save the known infos about the images
if
self
.
images_cache_filename
is
not
None
:
...
...
@@ -131,6 +134,12 @@ class Host(object):
return
attrs
[
"image"
]
def
dbenv2docker
(
self
,
key
):
"""Returns a nice docker image name given a BEAT database environment key"""
attrs
=
self
.
db_environments
[
key
]
return
attrs
[
"image"
]
def
teardown
(
self
):
for
container
in
self
.
containers
:
self
.
rm
(
container
)
...
...
@@ -312,7 +321,7 @@ class Host(object):
logger
.
debug
(
"Description not found for"
,
image
)
continue
key
=
description
[
"name"
]
+
" ("
+
description
[
"version"
]
+
")"
key
=
build_env_name
(
description
)
if
"databases"
in
description
:
if
(
key
in
db_environments
)
and
not
_must_replace
(
...
...
@@ -400,7 +409,7 @@ class Host(object):
continue
image_info
=
_parse_image_info
(
image
)
key
=
"{} ({})"
.
format
(
image_info
[
"name"
],
image_info
[
"version"
]
)
key
=
build_env_name
(
image_info
)
image_name
=
image_info
[
"image"
]
if
key
in
environments
:
...
...
beat/core/execution/docker.py
View file @
9af90457
...
...
@@ -48,6 +48,8 @@ import logging
import
requests
import
simplejson
as
json
from
collections
import
namedtuple
from
beat.backend.python.execution
import
MessageHandler
from
beat.backend.python.data
import
getAllFilenames
...
...
@@ -188,33 +190,79 @@ class DockerExecutor(RemoteExecutor):
# Modify the paths to the databases in the dumped configuration files
root_folder
=
os
.
path
.
join
(
databases_configuration_path
,
"prefix"
,
"databases"
)
database_paths
=
{}
DatabaseInfo
=
namedtuple
(
"DatabaseInfo"
,
[
"path"
,
"environment"
])
databases_infos
=
{}
for
db_name
in
self
.
databases
.
key
s
():
for
db_name
,
db_object
,
in
self
.
databases
.
item
s
():
json_path
=
os
.
path
.
join
(
root_folder
,
db_name
+
".json"
)
with
open
(
json_path
,
"r"
)
as
f
:
db_data
=
json
.
load
(
f
)
database_paths
[
db_name
]
=
db_data
[
"root_folder"
]
db_data
[
"root_folder"
]
=
os
.
path
.
join
(
"/databases"
,
db_name
)
system_path
=
db_data
[
"root_folder"
]
container_path
=
os
.
path
.
join
(
"/databases"
,
db_name
)
db_data
[
"root_folder"
]
=
container_path
with
open
(
json_path
,
"w"
)
as
f
:
json
.
dump
(
db_data
,
f
,
indent
=
4
)
# Determine the docker image to use for the databases
try
:
databases_environment
=
self
.
host
.
db2docker
(
database_paths
.
keys
())
except
Exception
:
raise
RuntimeError
(
"No environment found for the databases `%s' "
"- available environments are %s"
%
(
", "
.
join
(
database_paths
.
keys
()),
", "
.
join
(
self
.
host
.
db_environments
.
keys
()),
)
databases_infos
[
db_name
]
=
DatabaseInfo
(
system_path
,
utils
.
build_env_name
(
db_object
.
environment
)
)
databases_environment
=
None
requesting_environments
=
{
name
:
info
for
name
,
info
in
databases_infos
.
items
()
if
info
.
environment
is
not
None
}
if
requesting_environments
:
if
len
(
requesting_environments
)
!=
len
(
self
.
databases
):
raise
RuntimeError
(
"Selected databases ({}) are not all providing"
" an environment."
.
format
(
list
(
self
.
databases
.
keys
()))
)
requested_environments
=
{
info
.
environment
for
info
in
requesting_environments
.
values
()
if
info
.
environment
is
not
None
}
if
len
(
requested_environments
)
>
1
:
raise
RuntimeError
(
"Selected databases ({}) are requesting different environments,"
"only one is supported"
.
format
(
list
(
requesting_environments
.
keys
()))
)
# All databases are requesting the same environment
db_environment
=
next
(
iter
(
requested_environments
))
try
:
databases_environment
=
self
.
host
.
dbenv2docker
(
db_environment
)
except
Exception
:
raise
RuntimeError
(
"Environment {} not found - available environments are {}"
.
format
(
db_environment
,
list
(
self
.
host
.
db_environments
.
keys
())
)
)
if
not
databases_environment
:
# Determine the docker image to use for the databases
database_list
=
databases_infos
.
keys
()
try
:
databases_environment
=
self
.
host
.
db2docker
(
database_list
)
except
Exception
:
raise
RuntimeError
(
"No environment found for the databases `%s' "
"- available environments are %s"
%
(
", "
.
join
(
database_list
),
", "
.
join
(
self
.
host
.
db_environments
.
keys
()),
)
)
# Creation of the container
# Note: we only support one databases image loaded at the same time
database_port
=
utils
.
find_free_port
()
...
...
@@ -244,8 +292,8 @@ class DockerExecutor(RemoteExecutor):
)
databases_info
.
add_volume
(
self
.
cache
,
self
.
CONTAINER_CACHE_PATH
)
for
db_name
,
db_
path
in
database
_path
s
.
items
():
databases_info
.
add_volume
(
db_path
,
os
.
path
.
join
(
"/databases"
,
db_name
))
for
db_name
,
db_
info
in
database
s_info
s
.
items
():
databases_info
.
add_volume
(
db_
info
.
path
,
os
.
path
.
join
(
"/databases"
,
db_name
))
# Start the container
while
True
:
...
...
@@ -392,7 +440,7 @@ class DockerExecutor(RemoteExecutor):
)
# Determine the docker image to use for the processing
processing_environment
=
"%(name)s (%(version)s)"
%
self
.
data
[
"environment"
]
processing_environment
=
utils
.
build_env_name
(
self
.
data
[
"environment"
]
)
if
processing_environment
not
in
self
.
host
:
raise
RuntimeError
(
"Environment `%s' is not available on docker "
...
...
beat/core/schema/database/1.json
View file @
9af90457
...
...
@@ -19,6 +19,19 @@
"items"
:
{
"$ref"
:
"#/definitions/protocol"
}
},
"environment"
:
{
"type"
:
"object"
,
"properties"
:
{
"name"
:
{
"type"
:
"string"
},
"version"
:
{
"type"
:
"string"
}
},
"required"
:
[
"name"
,
"version"
],
"additionalProperties"
:
false
},
"description"
:
{
"$ref"
:
"../common/1.json#/definitions/description"
},
"schema_version"
:
{
"$ref"
:
"../common/1.json#/definitions/version"
}
...
...
beat/core/schema/database/2.json
View file @
9af90457
...
...
@@ -21,6 +21,19 @@
"description"
:
{
"$ref"
:
"../common/1.json#/definitions/description"
},
"environment"
:
{
"type"
:
"object"
,
"properties"
:
{
"name"
:
{
"type"
:
"string"
},
"version"
:
{
"type"
:
"string"
}
},
"required"
:
[
"name"
,
"version"
],
"additionalProperties"
:
false
},
"schema_version"
:
{
"const"
:
2
}
},
...
...
beat/core/test/prefix/databases/integers_db_invalid_env/1.json
0 → 100644
View file @
9af90457
{
"root_folder"
:
"/tmp/path/not/set"
,
"environment"
:
{
"name"
:
"Does not exist"
,
"version"
:
"1.4.0"
},
"protocols"
:
[
{
"name"
:
"double"
,
"template"
:
"double"
,
"sets"
:
[
{
"name"
:
"double"
,
"template"
:
"double"
,
"view"
:
"Double"
,
"outputs"
:
{
"a"
:
"user/single_integer/1"
,
"b"
:
"user/single_integer/1"
,
"sum"
:
"user/single_integer/1"
}
}
]
},
{
"name"
:
"triple"
,
"template"
:
"triple"
,
"sets"
:
[
{
"name"
:
"triple"
,
"view"
:
"Triple"
,
"template"
:
"triple"
,
"outputs"
:
{
"a"
:
"user/single_integer/1"
,
"b"
:
"user/single_integer/1"
,
"c"
:
"user/single_integer/1"
,
"sum"
:
"user/single_integer/1"
}
}
]
},
{
"name"
:
"two_sets"
,
"template"
:
"two_sets"
,
"sets"
:
[
{
"name"
:
"double"
,
"template"
:
"double"
,
"view"
:
"Double"
,
"outputs"
:
{
"a"
:
"user/single_integer/1"
,
"b"
:
"user/single_integer/1"
,
"sum"
:
"user/single_integer/1"
}
},
{
"name"
:
"triple"
,
"template"
:
"triple"
,
"view"
:
"Triple"
,
"outputs"
:
{
"a"
:
"user/single_integer/1"
,
"b"
:
"user/single_integer/1"
,
"c"
:
"user/single_integer/1"
,
"sum"
:
"user/single_integer/1"
}
}
]
},
{
"name"
:
"labelled"
,
"template"
:
"labelled"
,
"sets"
:
[
{
"name"
:
"labelled"
,
"template"
:
"labelled"
,
"view"
:
"Labelled"
,
"outputs"
:
{
"value"
:
"user/single_integer/1"
,
"label"
:
"user/single_string/1"
}
}
]
},
{
"name"
:
"different_frequencies"
,
"template"
:
"different_frequencies"
,
"sets"
:
[
{
"name"
:
"double"
,
"template"
:
"double"
,
"view"
:
"DifferentFrequencies"
,
"outputs"
:
{
"a"
:
"user/single_integer/1"
,
"b"
:
"user/single_integer/1"
}
}
]
}
]
}
beat/core/test/prefix/databases/integers_db_invalid_env/1.py
0 → 100755
View file @
9af90457
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import
numpy
from
collections
import
namedtuple
from
beat.backend.python.database
import
View
class
Double
(
View
):
def
index
(
self
,
root_folder
,
parameters
):
Entry
=
namedtuple
(
"Entry"
,
[
"a"
,
"b"
,
"sum"
])
return
[
Entry
(
1
,
10
,
11
),
Entry
(
2
,
20
,
22
),
Entry
(
3
,
30
,
33
),
Entry
(
4
,
40
,
44
),
Entry
(
5
,
50
,
55
),
Entry
(
6
,
60
,
66
),
Entry
(
7
,
70
,
77
),
Entry
(
8
,
80
,
88
),
Entry
(
9
,
90
,
99
),
]
def
get
(
self
,
output
,
index
):
obj
=
self
.
objs
[
index
]
if
output
==
"a"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
a
)}
elif
output
==
"b"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
b
)}
elif
output
==
"sum"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
sum
)}
elif
output
==
"class"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
cls
)}
# ----------------------------------------------------------
class
Triple
(
View
):
def
index
(
self
,
root_folder
,
parameters
):
Entry
=
namedtuple
(
"Entry"
,
[
"a"
,
"b"
,
"c"
,
"sum"
])
return
[
Entry
(
1
,
10
,
100
,
111
),
Entry
(
2
,
20
,
200
,
222
),
Entry
(
3
,
30
,
300
,
333
),
Entry
(
4
,
40
,
400
,
444
),
Entry
(
5
,
50
,
500
,
555
),
Entry
(
6
,
60
,
600
,
666
),
Entry
(
7
,
70
,
700
,
777
),
Entry
(
8
,
80
,
800
,
888
),
Entry
(
9
,
90
,
900
,
999
),
]
def
get
(
self
,
output
,
index
):
obj
=
self
.
objs
[
index
]
if
output
==
"a"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
a
)}
elif
output
==
"b"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
b
)}
elif
output
==
"c"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
c
)}
elif
output
==
"sum"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
sum
)}
# ----------------------------------------------------------
class
Labelled
(
View
):
def
index
(
self
,
root_folder
,
parameters
):
Entry
=
namedtuple
(
"Entry"
,
[
"label"
,
"value"
])
return
[
Entry
(
"A"
,
1
),
Entry
(
"A"
,
2
),
Entry
(
"A"
,
3
),
Entry
(
"A"
,
4
),
Entry
(
"A"
,
5
),
Entry
(
"B"
,
10
),
Entry
(
"B"
,
20
),
Entry
(
"B"
,
30
),
Entry
(
"B"
,
40
),
Entry
(
"B"
,
50
),
Entry
(
"C"
,
100
),
Entry
(
"C"
,
200
),
Entry
(
"C"
,
300
),
Entry
(
"C"
,
400
),
Entry
(
"C"
,
500
),
]
def
get
(
self
,
output
,
index
):
obj
=
self
.
objs
[
index
]
if
output
==
"label"
:
return
{
"value"
:
obj
.
label
}
elif
output
==
"value"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
value
)}
# ----------------------------------------------------------
class
DifferentFrequencies
(
View
):
def
index
(
self
,
root_folder
,
parameters
):
Entry
=
namedtuple
(
"Entry"
,
[
"a"
,
"b"
])
return
[
Entry
(
1
,
10
),
Entry
(
1
,
20
),
Entry
(
1
,
30
),
Entry
(
1
,
40
),
Entry
(
2
,
50
),
Entry
(
2
,
60
),
Entry
(
2
,
70
),
Entry
(
2
,
80
),
]
def
get
(
self
,
output
,
index
):
obj
=
self
.
objs
[
index
]
if
output
==
"a"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
a
)}
elif
output
==
"b"
:
return
{
"value"
:
numpy
.
int32
(
obj
.
b
)}
beat/core/test/prefix/databases/integers_db_invalid_env/2.json
0 → 100644
View file @
9af90457
{
"schema_version"
:
2
,
"environment"
:
{
"name"
:
"Does not exist"
,
"version"
:
"1.4.0"
},
"root_folder"
:
"/tmp/path/not/set"
,
"protocols"
:
[
{
"name"
:
"double"
,
"template"
:
"double/1"
,
"views"
:
{
"double"
:
{
"view"
:
"Double"
}
}
},
{
"name"
:
"triple"
,
"template"
:
"triple/1"
,
"views"
:
{
"triple"
:
{
"view"
:
"Triple"
}
}
},
{
"name"
:
"two_sets"
,
"template"
:
"two_sets/1"
,
"views"
:
{
"double"
:
{
"view"
:
"Double"
},
"triple"
:
{
"view"
:
"Triple"
}
}
},
{
"name"
:
"labelled"
,
"template"
:
"labelled/1"
,
"views"
:
{
"labelled"
:
{
"view"
:
"Labelled"
}
}
},
{
"name"
:
"different_frequencies"
,
"template"
:
"different_frequencies/1"
,
"views"
:
{
"double"
:
{
"view"
:
"DifferentFrequencies"
}
}
}
]
}
beat/core/test/prefix/databases/integers_db_invalid_env/2.py
0 → 100644
View file @
9af90457
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import
numpy
from
collections
import
namedtuple
from
beat.backend.python.database
import
View
class
Double
(
View
):
def
index
(
self
,
root_folder
,
parameters
):
Entry
=
namedtuple
(
"Entry"
,
[
"a"
,
"b"
,
"sum"
])