Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
beat.backend.python
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
beat
beat.backend.python
Commits
066ab1f9
Commit
066ab1f9
authored
7 years ago
by
Samuel GAIST
Browse files
Options
Downloads
Patches
Plain Diff
[algorithm] Improved doc and fixed related warnings
parent
084da96e
No related branches found
No related tags found
2 merge requests
!17
Merge development branch 1.5.x
,
!13
Improve documentation
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
beat/backend/python/algorithm.py
+43
-37
43 additions, 37 deletions
beat/backend/python/algorithm.py
with
43 additions
and
37 deletions
beat/backend/python/algorithm.py
+
43
−
37
View file @
066ab1f9
...
@@ -26,7 +26,13 @@
...
@@ -26,7 +26,13 @@
###############################################################################
###############################################################################
"""
Validation for algorithms
"""
"""
=========
algorithm
=========
Validation for algorithms
"""
import
os
import
os
import
sys
import
sys
...
@@ -41,7 +47,7 @@ from . import loader
...
@@ -41,7 +47,7 @@ from . import loader
from
.
import
utils
from
.
import
utils
#----------------------------------------------------------
#
----------------------------------------------------------
class
Storage
(
utils
.
CodeStorage
):
class
Storage
(
utils
.
CodeStorage
):
...
@@ -70,11 +76,11 @@ class Storage(utils.CodeStorage):
...
@@ -70,11 +76,11 @@ class Storage(utils.CodeStorage):
super
(
Storage
,
self
).
__init__
(
path
,
language
)
super
(
Storage
,
self
).
__init__
(
path
,
language
)
#----------------------------------------------------------
#
----------------------------------------------------------
class
Runner
(
object
):
class
Runner
(
object
):
'''
A special loader class for algorithms, with specialized methods
"""
A special loader class for algorithms, with specialized methods
Parameters:
Parameters:
...
@@ -91,8 +97,7 @@ class Runner(object):
...
@@ -91,8 +97,7 @@ class Runner(object):
translating the exception from the user code. Read the documentation of
translating the exception from the user code. Read the documentation of
:py:func:`.loader.run` for more details.
:py:func:`.loader.run` for more details.
'''
"""
def
__init__
(
self
,
module
,
obj_name
,
algorithm
,
exc
=
None
):
def
__init__
(
self
,
module
,
obj_name
,
algorithm
,
exc
=
None
):
...
@@ -103,7 +108,7 @@ class Runner(object):
...
@@ -103,7 +108,7 @@ class Runner(object):
type
,
value
,
traceback
=
sys
.
exc_info
()
type
,
value
,
traceback
=
sys
.
exc_info
()
six
.
reraise
(
exc
,
exc
(
value
),
traceback
)
six
.
reraise
(
exc
,
exc
(
value
),
traceback
)
else
:
else
:
raise
# Just re-raise the user exception
raise
# Just re-raise the user exception
self
.
obj
=
loader
.
run
(
class_
,
'
__new__
'
,
exc
)
self
.
obj
=
loader
.
run
(
class_
,
'
__new__
'
,
exc
)
self
.
name
=
module
.
__name__
self
.
name
=
module
.
__name__
...
@@ -146,7 +151,7 @@ class Runner(object):
...
@@ -146,7 +151,7 @@ class Runner(object):
exc
=
self
.
exc
or
ValueError
exc
=
self
.
exc
or
ValueError
raise
exc
(
message
)
raise
exc
(
message
)
else
:
# User has not set a value, use the default
else
:
# User has not set a value, use the default
value
=
algo_parameters
[
key
][
'
default
'
]
value
=
algo_parameters
[
key
][
'
default
'
]
# In the end, set the value on the dictionary to be returned
# In the end, set the value on the dictionary to be returned
...
@@ -156,7 +161,7 @@ class Runner(object):
...
@@ -156,7 +161,7 @@ class Runner(object):
def
setup
(
self
,
parameters
):
def
setup
(
self
,
parameters
):
'''
Sets up the algorithm, only effective on the first call
'''
"""
Sets up the algorithm, only effective on the first call
"""
# Only effective on the first call
# Only effective on the first call
if
self
.
ready
:
if
self
.
ready
:
...
@@ -169,7 +174,7 @@ class Runner(object):
...
@@ -169,7 +174,7 @@ class Runner(object):
def
prepare
(
self
,
data_loaders
):
def
prepare
(
self
,
data_loaders
):
'''
Let the algorithm process the data on the non-principal channels
'''
"""
Let the algorithm process the data on the non-principal channels
"""
# Only effective on the first call
# Only effective on the first call
if
self
.
prepared
:
if
self
.
prepared
:
...
@@ -198,7 +203,7 @@ class Runner(object):
...
@@ -198,7 +203,7 @@ class Runner(object):
def
process
(
self
,
inputs
=
None
,
data_loaders
=
None
,
outputs
=
None
,
output
=
None
):
def
process
(
self
,
inputs
=
None
,
data_loaders
=
None
,
outputs
=
None
,
output
=
None
):
'''
Runs through data
'''
"""
Runs through data
"""
exc
=
self
.
exc
or
RuntimeError
exc
=
self
.
exc
or
RuntimeError
...
@@ -243,11 +248,12 @@ class Runner(object):
...
@@ -243,11 +248,12 @@ class Runner(object):
def
__getattr__
(
self
,
key
):
def
__getattr__
(
self
,
key
):
'''
Returns an attribute of the algorithm - only called at last resort
'''
"""
Returns an attribute of the algorithm - only called at last resort
"""
return
getattr
(
self
.
obj
,
key
)
return
getattr
(
self
.
obj
,
key
)
#----------------------------------------------------------
#
----------------------------------------------------------
class
Algorithm
(
object
):
class
Algorithm
(
object
):
...
@@ -255,8 +261,8 @@ class Algorithm(object):
...
@@ -255,8 +261,8 @@ class Algorithm(object):
This class can only parse the meta-parameters of the algorithm (i.e., input
This class can only parse the meta-parameters of the algorithm (i.e., input
and output declaration, grouping, synchronization details, parameters and
and output declaration, grouping, synchronization details, parameters and
splittability). The actual algorithm is not directly treated by this class
-
splittability). The actual algorithm is not directly treated by this class
.
i
t can, however, provide you with a loader for actually running the
I
t can, however, provide you with a loader for actually running the
algorithmic code (see :py:meth:`Algorithm.runner`).
algorithmic code (see :py:meth:`Algorithm.runner`).
...
@@ -281,33 +287,34 @@ class Algorithm(object):
...
@@ -281,33 +287,34 @@ class Algorithm(object):
name (str): The algorithm name
name (str): The algorithm name
dataformats (dict): A dictionary containing all pre-loaded dataformats
used
dataformats (dict): A dictionary containing all pre-loaded dataformats
by this algorithm. Data format objects will be of type
used
by this algorithm. Data format objects will be of type
:py:class:`.dataformat.DataFormat`.
:py:class:`.dataformat.DataFormat`.
libraries (dict): A mapping object defining other libraries this
algorithm
libraries (dict): A mapping object defining other libraries this
needs to load so it can work properly.
algorithm
needs to load so it can work properly.
uses (dict): A mapping object defining the required library import name
uses (dict): A mapping object defining the required library import name
(keys) and the full-names (values).
(keys) and the full-names (values).
parameters (dict): A dictionary containing all pre-defined parameters
that
parameters (dict): A dictionary containing all pre-defined parameters
this algorithm accepts.
that
this algorithm accepts.
splittable (bool): A boolean value that indicates if this algorithm is
splittable (bool): A boolean value that indicates if this algorithm is
automatically parallelizeable by our backend.
automatically parallelizeable by our backend.
input_map (dict): A dictionary where the key is the input name and the
input_map (dict): A dictionary where the key is the input name and the
value, its type. All input names (potentially from different groups)
are
value, its type. All input names (potentially from different groups)
comprised in this dictionary.
are
comprised in this dictionary.
output_map (dict): A dictionary where the key is the output name and the
output_map (dict): A dictionary where the key is the output name and the
value, its type. All output names (potentially from different groups)
are
value, its type. All output names (potentially from different groups)
comprised in this dictionary.
are
comprised in this dictionary.
results (dict): If this algorithm is actually an analyzer (i.e., there are
results (dict): If this algorithm is actually an analyzer (i.e., there
no formal outputs, but results that must be saved by the platform), then
are no formal outputs, but results that must be saved by the platform),
this dictionary contains the names and data types of those elements.
then this dictionary contains the names and data types of those
elements.
groups (dict): A list containing dictionaries with inputs and outputs
groups (dict): A list containing dictionaries with inputs and outputs
belonging to the same synchronization group.
belonging to the same synchronization group.
...
@@ -385,9 +392,9 @@ class Algorithm(object):
...
@@ -385,9 +392,9 @@ class Algorithm(object):
if
dataformat_cache
and
input
[
'
type
'
]
in
dataformat_cache
:
#reuse
if
dataformat_cache
and
input
[
'
type
'
]
in
dataformat_cache
:
#reuse
thisformat
=
dataformat_cache
[
input
[
'
type
'
]]
thisformat
=
dataformat_cache
[
input
[
'
type
'
]]
else
:
#
load it
else
:
#
load it
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
input
[
'
type
'
])
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
input
[
'
type
'
])
if
dataformat_cache
is
not
None
:
#
update it
if
dataformat_cache
is
not
None
:
#
update it
dataformat_cache
[
input
[
'
type
'
]]
=
thisformat
dataformat_cache
[
input
[
'
type
'
]]
=
thisformat
self
.
dataformats
[
input
[
'
type
'
]]
=
thisformat
self
.
dataformats
[
input
[
'
type
'
]]
=
thisformat
...
@@ -397,9 +404,9 @@ class Algorithm(object):
...
@@ -397,9 +404,9 @@ class Algorithm(object):
for
name
,
output
in
group
[
'
outputs
'
].
items
():
for
name
,
output
in
group
[
'
outputs
'
].
items
():
if
output
[
'
type
'
]
in
self
.
dataformats
:
continue
if
output
[
'
type
'
]
in
self
.
dataformats
:
continue
if
dataformat_cache
and
output
[
'
type
'
]
in
dataformat_cache
:
#
reuse
if
dataformat_cache
and
output
[
'
type
'
]
in
dataformat_cache
:
#
reuse
thisformat
=
dataformat_cache
[
output
[
'
type
'
]]
thisformat
=
dataformat_cache
[
output
[
'
type
'
]]
else
:
#
load it
else
:
#
load it
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
output
[
'
type
'
])
thisformat
=
dataformat
.
DataFormat
(
self
.
prefix
,
output
[
'
type
'
])
if
dataformat_cache
is
not
None
:
#update it
if
dataformat_cache
is
not
None
:
#update it
dataformat_cache
[
output
[
'
type
'
]]
=
thisformat
dataformat_cache
[
output
[
'
type
'
]]
=
thisformat
...
@@ -550,8 +557,8 @@ class Algorithm(object):
...
@@ -550,8 +557,8 @@ class Algorithm(object):
"""
Checks if a given value against a declared parameter
"""
Checks if a given value against a declared parameter
This method checks if the provided user value can be safe-cast to the
This method checks if the provided user value can be safe-cast to the
parameter type as defined on its specification and that it conforms to
any
parameter type as defined on its specification and that it conforms to
parameter-imposed restrictions.
any
parameter-imposed restrictions.
Parameters:
Parameters:
...
@@ -573,10 +580,9 @@ class Algorithm(object):
...
@@ -573,10 +580,9 @@ class Algorithm(object):
declaration.
declaration.
ValueError: If the parameter cannot be safe cast into the algorithm
'
s
ValueError: If the parameter cannot be safe cast into the algorithm
'
s
type. Alternatively, a ``ValueError`` may also be raised if a range
or
type. Alternatively, a ``ValueError`` may also be raised if a range
choice was specified and the value does not obey those settings
or
choice was specified and the value does not obey those settings
stipulated for the parameter
stipulated for the parameter
"""
"""
if
(
self
.
parameters
is
None
)
or
(
parameter
not
in
self
.
parameters
):
if
(
self
.
parameters
is
None
)
or
(
parameter
not
in
self
.
parameters
):
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment