Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.core
Commits
91252520
Commit
91252520
authored
May 29, 2018
by
Samuel GAIST
Browse files
[algorithm] Fixed documentation related warnings
Also cleaned unused imports
parent
6a05f6b5
Changes
1
Hide whitespace changes
Inline
Side-by-side
beat/core/algorithm.py
View file @
91252520
...
...
@@ -25,7 +25,6 @@
# #
###############################################################################
"""
=========
algorithm
...
...
@@ -34,12 +33,9 @@ algorithm
Validation for algorithms
"""
import
os
import
sys
import
six
import
numpy
import
simplejson
from
.
import
dataformat
from
.
import
library
...
...
@@ -52,14 +48,13 @@ from beat.backend.python.algorithm import Runner
from
beat.backend.python.algorithm
import
Algorithm
as
BackendAlgorithm
class
Algorithm
(
BackendAlgorithm
):
"""Algorithms represent runnable components within the platform.
This class can only parse the meta-parameters of the algorithm (i.e., input
and output declaration, grouping, synchronization details, parameters and
splittability). The actual algorithm is not directly treated by this class
-
i
t can, however, provide you with a loader for actually running the
splittability). The actual algorithm is not directly treated by this class
.
I
t can, however, provide you with a loader for actually running the
algorithmic code (see :py:meth:`.runner`).
...
...
@@ -99,33 +94,34 @@ class Algorithm(BackendAlgorithm):
storage (object): A simple object that provides information about file
paths for this algorithm
dataformats (dict): A dictionary containing all pre-loaded dataformats
used
by this algorithm. Data format objects will be of type
dataformats (dict): A dictionary containing all pre-loaded dataformats
used
by this algorithm. Data format objects will be of type
:py:class:`beat.core.dataformat.DataFormat`.
libraries (dict): A mapping object defining other libraries this
algorithm
needs to load so it can work properly.
libraries (dict): A mapping object defining other libraries this
algorithm
needs to load so it can work properly.
uses (dict): A mapping object defining the required library import name
(keys) and the full-names (values).
parameters (dict): A dictionary containing all pre-defined parameters
that
this algorithm accepts.
parameters (dict): A dictionary containing all pre-defined parameters
that
this algorithm accepts.
splittable (bool): A boolean value that indicates if this algorithm is
automatically parallelizeable by our backend.
input_map (dict): A dictionary where the key is the input name and the
value, its type. All input names (potentially from different groups)
are
comprised in this dictionary.
value, its type. All input names (potentially from different groups)
are
comprised in this dictionary.
output_map (dict): A dictionary where the key is the output name and the
value, its type. All output names (potentially from different groups)
are
comprised in this dictionary.
value, its type. All output names (potentially from different groups)
are
comprised in this dictionary.
results (dict): If this algorithm is actually an analyzer (i.e., there are
no formal outputs, but results that must be saved by the platform), then
this dictionary contains the names and data types of those elements.
results (dict): If this algorithm is actually an analyzer (i.e., there
are no formal outputs, but results that must be saved by the platform),
then this dictionary contains the names and data types of those
elements.
groups (dict): A list containing dictionaries with inputs and outputs
belonging to the same synchronization group.
...
...
@@ -154,21 +150,21 @@ class Algorithm(BackendAlgorithm):
self
.
_name
=
None
self
.
storage
=
None
self
.
dataformats
=
{}
# preloaded dataformats
self
.
libraries
=
{}
# preloaded libraries
self
.
dataformats
=
{}
# preloaded dataformats
self
.
libraries
=
{}
# preloaded libraries
code
=
None
if
data
is
None
:
#
loads prototype and validates it
if
data
is
None
:
#
loads prototype and validates it
data
=
None
code
=
None
elif
isinstance
(
data
,
(
tuple
,
list
)):
#
user has passed individual info
elif
isinstance
(
data
,
(
tuple
,
list
)):
#
user has passed individual info
data
,
code
=
data
#
break down into two components
data
,
code
=
data
#
break down into two components
if
isinstance
(
data
,
six
.
string_types
):
#
user has passed a file pointer
if
isinstance
(
data
,
six
.
string_types
):
#
user has passed a file pointer
self
.
_name
=
data
self
.
storage
=
Storage
(
self
.
prefix
,
self
.
_name
)
...
...
@@ -176,21 +172,21 @@ class Algorithm(BackendAlgorithm):
self
.
errors
.
append
(
'Algorithm declaration file not found: %s'
%
data
)
return
data
=
self
.
storage
.
json
.
path
#
loads data from JSON declaration
data
=
self
.
storage
.
json
.
path
#
loads data from JSON declaration
# At this point, `data' can be a dictionary or ``None``
if
data
is
None
:
# loads the default declaration for an algorithm
if
data
is
None
:
# loads the default declaration for an algorithm
self
.
data
,
self
.
errors
=
prototypes
.
load
(
'algorithm'
)
assert
not
self
.
errors
,
"
\n
* %s"
%
"
\n
*"
.
join
(
self
.
errors
)
else
:
# just assign it
else
:
# just assign it
# this runs basic validation, including JSON loading if required
self
.
data
,
self
.
errors
=
schema
.
validate
(
'algorithm'
,
data
)
if
self
.
errors
:
return
#
don't proceed with the rest of validation
if
self
.
errors
:
return
#
don't proceed with the rest of validation
if
self
.
storage
is
not
None
:
#
loading from the disk, check code
if
self
.
storage
is
not
None
:
#
loading from the disk, check code
if
not
self
.
storage
.
code
.
exists
():
if
self
.
data
[
'language'
]
!=
'cxx'
:
self
.
errors
.
append
(
'Algorithm code not found: %s'
%
\
...
...
@@ -201,15 +197,15 @@ class Algorithm(BackendAlgorithm):
# At this point, `code' can be a string (or a binary blob) or ``None``
if
code
is
None
:
# loads the default code for an algorithm
if
code
is
None
:
# loads the default code for an algorithm
self
.
code
=
prototypes
.
binary_load
(
'algorithm.py'
)
self
.
data
[
'language'
]
=
'python'
else
:
# just assign it - notice that in this case, no language is set
else
:
# just assign it - notice that in this case, no language is set
self
.
code
=
code
if
self
.
errors
:
return
#
don't proceed with the rest of validation
if
self
.
errors
:
return
#
don't proceed with the rest of validation
# if no errors so far, make sense out of the declaration data
...
...
@@ -261,11 +257,11 @@ class Algorithm(BackendAlgorithm):
for
name
,
input
in
group
[
'inputs'
].
items
():
if
input
[
'type'
]
in
self
.
dataformats
:
continue
if
dataformat_cache
and
input
[
'type'
]
in
dataformat_cache
:
#
reuse
if
dataformat_cache
and
input
[
'type'
]
in
dataformat_cache
:
#
reuse
thisformat
=
dataformat_cache
[
input
[
'type'
]]
else
:
#
load it
else
:
#
load it
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
input
[
'type'
])
if
dataformat_cache
is
not
None
:
#
update it
if
dataformat_cache
is
not
None
:
#
update it
dataformat_cache
[
input
[
'type'
]]
=
thisformat
self
.
dataformats
[
input
[
'type'
]]
=
thisformat
...
...
@@ -281,11 +277,11 @@ class Algorithm(BackendAlgorithm):
for
name
,
output
in
group
[
'outputs'
].
items
():
if
output
[
'type'
]
in
self
.
dataformats
:
continue
if
dataformat_cache
and
output
[
'type'
]
in
dataformat_cache
:
#
reuse
if
dataformat_cache
and
output
[
'type'
]
in
dataformat_cache
:
#
reuse
thisformat
=
dataformat_cache
[
output
[
'type'
]]
else
:
#
load it
else
:
#
load it
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
output
[
'type'
])
if
dataformat_cache
is
not
None
:
#
update it
if
dataformat_cache
is
not
None
:
#
update it
dataformat_cache
[
output
[
'type'
]]
=
thisformat
self
.
dataformats
[
output
[
'type'
]]
=
thisformat
...
...
@@ -304,11 +300,11 @@ class Algorithm(BackendAlgorithm):
if
result
[
'type'
]
in
self
.
dataformats
:
continue
if
dataformat_cache
and
result
[
'type'
]
in
dataformat_cache
:
#
reuse
if
dataformat_cache
and
result
[
'type'
]
in
dataformat_cache
:
#
reuse
thisformat
=
dataformat_cache
[
result
[
'type'
]]
else
:
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
result
[
'type'
])
if
dataformat_cache
is
not
None
:
#
update it
if
dataformat_cache
is
not
None
:
#
update it
dataformat_cache
[
result
[
'type'
]]
=
thisformat
self
.
dataformats
[
result
[
'type'
]]
=
thisformat
...
...
@@ -321,7 +317,8 @@ class Algorithm(BackendAlgorithm):
def
_convert_parameter_types
(
self
):
"""Converts types to numpy equivalents, checks defaults, ranges and choices
"""Converts types to numpy equivalents, checks defaults, ranges and
choices
"""
def
_try_convert
(
name
,
tp
,
value
,
desc
):
...
...
@@ -358,14 +355,14 @@ class Algorithm(BackendAlgorithm):
parameter
[
'default'
]
=
_try_convert
(
name
,
parameter
[
'type'
],
parameter
[
'default'
],
'default'
)
if
'range'
in
parameter
:
#
check range
if
'range'
in
parameter
:
#
check range
if
parameter
[
'default'
]
<
parameter
[
'range'
][
0
]
or
\
parameter
[
'default'
]
>
parameter
[
'range'
][
1
]:
self
.
errors
.
append
(
"default for parameter `%s' (%r) is not "
\
"within parameter range [%r, %r]"
%
(
name
,
parameter
[
'default'
],
parameter
[
'range'
][
0
],
parameter
[
'range'
][
1
]))
if
'choice'
in
parameter
:
#
check choices
if
'choice'
in
parameter
:
#
check choices
if
parameter
[
'default'
]
not
in
parameter
[
'choice'
]:
self
.
errors
.
append
(
"default for parameter `%s' (%r) is not "
\
"a valid choice `[%s]'"
%
(
name
,
parameter
[
'default'
],
...
...
@@ -391,13 +388,13 @@ class Algorithm(BackendAlgorithm):
def
_check_language_consistence
(
self
):
# all used libraries must be programmed with the same language
if
self
.
language
==
'unknown'
:
return
#
bail out on unknown language
if
self
.
language
==
'unknown'
:
return
#
bail out on unknown language
if
self
.
uses
:
for
name
,
library
in
self
.
uses
.
items
():
if
library
not
in
self
.
libraries
:
continue
#
invalid
if
library
not
in
self
.
libraries
:
continue
#
invalid
if
self
.
libraries
[
library
].
data
is
None
:
self
.
errors
.
append
(
"language for used library `%s' cannot be "
\
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment