Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.backend.python
Commits
54aa50bd
Commit
54aa50bd
authored
Mar 22, 2017
by
Philip ABBET
Browse files
Refactoring of the 'Library' and 'Algorithm' classes (which were duplicated in beat.core)
parent
235798e9
Changes
2
Hide whitespace changes
Inline
Side-by-side
beat/backend/python/algorithm.py
View file @
54aa50bd
...
...
@@ -38,6 +38,34 @@ import simplejson
from
.
import
dataformat
from
.
import
library
from
.
import
loader
from
.
import
utils
class
Storage
(
utils
.
CodeStorage
):
"""Resolves paths for algorithms
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the algorithm object in the format
``<user>/<name>/<version>``.
"""
def
__init__
(
self
,
prefix
,
name
,
language
=
None
):
if
name
.
count
(
'/'
)
!=
2
:
raise
RuntimeError
(
"invalid algorithm name: `%s'"
%
name
)
self
.
username
,
self
.
name
,
self
.
version
=
name
.
split
(
'/'
)
self
.
prefix
=
prefix
self
.
fullname
=
name
path
=
utils
.
hashed_or_simple
(
self
.
prefix
,
'algorithms'
,
name
)
super
(
Storage
,
self
).
__init__
(
path
,
language
)
class
Runner
(
object
):
...
...
@@ -160,6 +188,7 @@ class Runner(object):
return
getattr
(
self
.
obj
,
key
)
class
Algorithm
(
object
):
"""Algorithms represent runnable components within the platform.
...
...
@@ -222,6 +251,9 @@ class Algorithm(object):
groups (dict): A list containing dictionaries with inputs and outputs
belonging to the same synchronization group.
errors (list): A list containing errors found while loading this
algorithm.
data (dict): The original data for this algorithm, as loaded by our JSON
decoder.
...
...
@@ -232,20 +264,34 @@ class Algorithm(object):
def
__init__
(
self
,
prefix
,
name
,
dataformat_cache
=
None
,
library_cache
=
None
):
self
.
_name
=
None
self
.
storage
=
None
self
.
prefix
=
prefix
self
.
dataformats
=
{}
self
.
libraries
=
{}
self
.
groups
=
[]
dataformat_cache
=
dataformat_cache
if
dataformat_cache
is
not
None
else
{}
library_cache
=
library_cache
if
library_cache
is
not
None
else
{}
self
.
name
=
name
json_path
=
os
.
path
.
join
(
prefix
,
'algorithms'
,
name
+
'.json'
)
with
open
(
json_path
,
'rb'
)
as
f
:
self
.
data
=
simplejson
.
load
(
f
)
self
.
_load
(
name
,
dataformat_cache
,
library_cache
)
self
.
code_path
=
os
.
path
.
join
(
prefix
,
'algorithms'
,
name
+
'.py'
)
def
_load
(
self
,
data
,
dataformat_cache
,
library_cache
):
"""Loads the algorithm"""
self
.
_name
=
data
self
.
storage
=
Storage
(
self
.
prefix
,
data
)
json_path
=
self
.
storage
.
json
.
path
if
not
self
.
storage
.
exists
():
self
.
errors
.
append
(
'Algorithm declaration file not found: %s'
%
json_path
)
return
with
open
(
json_path
,
'rb'
)
as
f
:
self
.
data
=
simplejson
.
load
(
f
)
self
.
code_path
=
self
.
storage
.
code
.
path
self
.
groups
=
self
.
data
[
'groups'
]
...
...
@@ -375,6 +421,22 @@ class Algorithm(object):
library
.
Library
(
self
.
prefix
,
value
,
library_cache
))
@
property
def
name
(
self
):
"""Returns the name of this object
"""
return
self
.
_name
or
'__unnamed_algorithm__'
@
name
.
setter
def
name
(
self
,
value
):
if
self
.
data
[
'language'
]
==
'unknown'
:
raise
RuntimeError
(
"algorithm has no programming language set"
)
self
.
_name
=
value
self
.
storage
=
Storage
(
self
.
prefix
,
value
,
self
.
data
[
'language'
])
@
property
def
schema_version
(
self
):
...
...
@@ -382,6 +444,20 @@ class Algorithm(object):
return
self
.
data
.
get
(
'schema_version'
,
1
)
@
property
def
language
(
self
):
"""Returns the current language set for the executable code"""
return
self
.
data
[
'language'
]
@
language
.
setter
def
language
(
self
,
value
):
"""Sets the current executable code programming language"""
if
self
.
storage
:
self
.
storage
.
language
=
value
self
.
data
[
'language'
]
=
value
def
clean_parameter
(
self
,
parameter
,
value
):
"""Checks if a given value against a declared parameter
...
...
@@ -410,8 +486,8 @@ class Algorithm(object):
ValueError: If the parameter cannot be safe cast into the algorithm's
type. Alternatively, a ``ValueError`` may also be raised if a range or
choice was specified and the value does not ob
b
ey those settings
e
stipulated for the parameter
choice was specified and the value does not obey those settings
stipulated for the parameter
"""
...
...
@@ -437,35 +513,72 @@ class Algorithm(object):
return
retval
@
property
def
valid
(
self
):
"""A boolean that indicates if this algorithm is valid or not"""
return
not
bool
(
self
.
errors
)
@
property
def
uses
(
self
):
return
self
.
data
.
get
(
'uses'
)
@
uses
.
setter
def
uses
(
self
,
value
):
self
.
data
[
'uses'
]
=
value
return
value
@
property
def
results
(
self
):
return
self
.
data
.
get
(
'results'
)
@
results
.
setter
def
results
(
self
,
value
):
self
.
data
[
'results'
]
=
value
return
value
@
property
def
parameters
(
self
):
return
self
.
data
.
get
(
'parameters'
)
@
parameters
.
setter
def
parameters
(
self
,
value
):
self
.
data
[
'parameters'
]
=
value
return
value
@
property
def
splittable
(
self
):
return
self
.
data
.
get
(
'splittable'
,
False
)
@
splittable
.
setter
def
splittable
(
self
,
value
):
self
.
data
[
'splittable'
]
=
value
return
value
def
uses_dict
(
self
):
"""Returns the usage dictionary for all dependent modules"""
if
self
.
data
[
'language'
]
==
'unknown'
:
raise
RuntimeError
(
"algorithm has no programming language set"
)
if
not
self
.
_name
:
raise
RuntimeError
(
"algorithm has no name"
)
retval
=
{}
if
self
.
uses
is
not
None
:
for
name
,
value
in
self
.
uses
.
items
():
retval
[
name
]
=
dict
(
path
=
self
.
libraries
[
value
].
code
_
path
,
path
=
self
.
libraries
[
value
].
storage
.
code
.
path
,
uses
=
self
.
libraries
[
value
].
uses_dict
(),
)
...
...
@@ -489,11 +602,24 @@ class Algorithm(object):
before using the ``process`` method.
"""
if
not
self
.
_name
:
exc
=
exc
or
RuntimeError
raise
exc
(
"algorithm has no name"
)
if
self
.
data
[
'language'
]
==
'unknown'
:
exc
=
exc
or
RuntimeError
raise
exc
(
"algorithm has no programming language set"
)
if
not
self
.
valid
:
message
=
"cannot load code for invalid algorithm (%s)"
%
(
self
.
name
,)
exc
=
exc
or
RuntimeError
raise
exc
(
message
)
# loads the module only once through the lifetime of the algorithm object
try
:
self
.
__module
=
getattr
(
self
,
'module'
,
loader
.
load_module
(
self
.
name
.
replace
(
os
.
sep
,
'_'
),
self
.
code
_
path
,
self
.
uses_dict
()))
self
.
storage
.
code
.
path
,
self
.
uses_dict
()))
except
Exception
as
e
:
if
exc
is
not
None
:
type
,
value
,
traceback
=
sys
.
exc_info
()
...
...
@@ -504,6 +630,52 @@ class Algorithm(object):
return
Runner
(
self
.
__module
,
klass
,
self
,
exc
)
@
property
def
description
(
self
):
"""The short description for this object"""
return
self
.
data
.
get
(
'description'
,
None
)
@
description
.
setter
def
description
(
self
,
value
):
"""Sets the short description for this object"""
self
.
data
[
'description'
]
=
value
@
property
def
documentation
(
self
):
"""The full-length description for this object"""
if
not
self
.
_name
:
raise
RuntimeError
(
"algorithm has no name"
)
if
self
.
storage
.
doc
.
exists
():
return
self
.
storage
.
doc
.
load
()
return
None
@
documentation
.
setter
def
documentation
(
self
,
value
):
"""Sets the full-length description for this object"""
if
not
self
.
_name
:
raise
RuntimeError
(
"algorithm has no name"
)
if
hasattr
(
value
,
'read'
):
self
.
storage
.
doc
.
save
(
value
.
read
())
else
:
self
.
storage
.
doc
.
save
(
value
)
def
hash
(
self
):
"""Returns the hexadecimal hash for the current algorithm"""
if
not
self
.
_name
:
raise
RuntimeError
(
"algorithm has no name"
)
return
self
.
storage
.
hash
()
def
result_dataformat
(
self
):
"""Generates, on-the-fly, the dataformat for the result readout"""
...
...
beat/backend/python/library.py
View file @
54aa50bd
...
...
@@ -33,6 +33,34 @@ import os
import
simplejson
from
.
import
loader
from
.
import
utils
class
Storage
(
utils
.
CodeStorage
):
"""Resolves paths for libraries
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the library object in the format
``<user>/<name>/<version>``.
"""
def
__init__
(
self
,
prefix
,
name
,
language
=
None
):
if
name
.
count
(
'/'
)
!=
2
:
raise
RuntimeError
(
"invalid library name: `%s'"
%
name
)
self
.
username
,
self
.
name
,
self
.
version
=
name
.
split
(
'/'
)
self
.
prefix
=
prefix
self
.
fullname
=
name
path
=
utils
.
hashed_or_simple
(
self
.
prefix
,
'libraries'
,
name
)
super
(
Storage
,
self
).
__init__
(
path
,
language
)
class
Library
(
object
):
...
...
@@ -59,12 +87,23 @@ class Library(object):
name (str): The library name
description (str): The short description string, loaded from the JSON
file if one was set.
documentation (str): The full-length docstring for this object.
storage (object): A simple object that provides information about file
paths for this library
libraries (dict): A mapping object defining other libraries this library
needs to load so it can work properly.
uses (dict): A mapping object defining the required library import name
(keys) and the full-names (values).
errors (list): A list containing errors found while loading this
library.
data (dict): The original data for this library, as loaded by our JSON
decoder.
...
...
@@ -75,17 +114,36 @@ class Library(object):
def
__init__
(
self
,
prefix
,
name
,
library_cache
=
None
):
self
.
_name
=
None
self
.
storage
=
None
self
.
prefix
=
prefix
self
.
libraries
=
{}
library_cache
=
library_cache
if
library_cache
is
not
None
else
{}
self
.
name
=
name
json_path
=
os
.
path
.
join
(
prefix
,
'libraries'
,
name
+
'.json'
)
with
open
(
json_path
,
'rb'
)
as
f
:
self
.
data
=
simplejson
.
load
(
f
)
try
:
self
.
_load
(
name
,
library_cache
)
finally
:
if
self
.
_name
is
not
None
:
#registers it into the cache, even if failed
library_cache
[
self
.
_name
]
=
self
def
_load
(
self
,
data
,
library_cache
):
"""Loads the library"""
self
.
_name
=
name
self
.
storage
=
Storage
(
self
.
prefix
,
data
)
json_path
=
self
.
storage
.
json
.
path
if
not
self
.
storage
.
exists
():
self
.
errors
.
append
(
'Library declaration file not found: %s'
%
json_path
)
return
with
open
(
json_path
,
'rb'
)
as
f
:
self
.
data
=
simplejson
.
load
(
f
)
self
.
code_path
=
os
.
path
.
join
(
prefix
,
'libraries'
,
name
+
'.py'
)
self
.
code_path
=
self
.
storage
.
code
.
path
# if no errors so far, make sense out of the library data
self
.
data
.
setdefault
(
'uses'
,
{})
...
...
@@ -94,19 +152,25 @@ class Library(object):
for
name
,
value
in
self
.
uses
.
items
():
self
.
libraries
[
value
]
=
Library
(
self
.
prefix
,
value
,
library_cache
)
self
.
libraries
[
self
.
name
]
=
self
self
.
libraries
[
self
.
_
name
]
=
self
def
uses_dict
(
self
):
"""Returns the usage dictionary for all dependent modules"""
if
self
.
data
[
'language'
]
==
'unknown'
:
raise
RuntimeError
(
"library has no programming language set"
)
if
not
self
.
_name
:
raise
RuntimeError
(
"library has no name"
)
retval
=
{}
if
self
.
uses
is
not
None
:
for
name
,
value
in
self
.
uses
.
items
():
retval
[
name
]
=
dict
(
path
=
self
.
libraries
[
value
].
code
_
path
,
path
=
self
.
libraries
[
value
].
storage
.
code
.
path
,
uses
=
self
.
libraries
[
value
].
uses_dict
(),
)
...
...
@@ -119,8 +183,31 @@ class Library(object):
Returns the loaded Python module.
"""
if
self
.
data
[
'language'
]
==
'unknown'
:
raise
RuntimeError
(
"library has no programming language set"
)
if
not
self
.
_name
:
raise
RuntimeError
(
"library has no name"
)
return
loader
.
load_module
(
self
.
name
.
replace
(
os
.
sep
,
'_'
),
self
.
code_path
,
self
.
uses_dict
())
self
.
storage
.
code
.
path
,
self
.
uses_dict
())
@
property
def
name
(
self
):
"""Returns the name of this object
"""
return
self
.
_name
or
'__unnamed_library__'
@
name
.
setter
def
name
(
self
,
value
):
if
self
.
data
[
'language'
]
==
'unknown'
:
raise
RuntimeError
(
"library has no programming language set"
)
self
.
_name
=
value
self
.
storage
=
Storage
(
self
.
prefix
,
value
,
self
.
data
[
'language'
])
@
property
...
...
@@ -129,6 +216,79 @@ class Library(object):
return
self
.
data
.
get
(
'schema_version'
,
1
)
@
property
def
language
(
self
):
"""Returns the current language set for the library code"""
return
self
.
data
[
'language'
]
@
language
.
setter
def
language
(
self
,
value
):
"""Sets the current executable code programming language"""
if
self
.
storage
:
self
.
storage
.
language
=
value
self
.
data
[
'language'
]
=
value
self
.
_check_language_consistence
()
@
property
def
valid
(
self
):
"""A boolean that indicates if this library is valid or not"""
return
not
bool
(
self
.
errors
)
@
property
def
uses
(
self
):
return
self
.
data
.
get
(
'uses'
)
@
uses
.
setter
def
uses
(
self
,
value
):
self
.
data
[
'uses'
]
=
value
return
value
@
property
def
description
(
self
):
"""The short description for this object"""
return
self
.
data
.
get
(
'description'
,
None
)
@
description
.
setter
def
description
(
self
,
value
):
"""Sets the short description for this object"""
self
.
data
[
'description'
]
=
value
@
property
def
documentation
(
self
):
"""The full-length description for this object"""
if
not
self
.
_name
:
raise
RuntimeError
(
"library has no name"
)
if
self
.
storage
.
doc
.
exists
():
return
self
.
storage
.
doc
.
load
()
return
None
@
documentation
.
setter
def
documentation
(
self
,
value
):
"""Sets the full-length description for this object"""
if
not
self
.
_name
:
raise
RuntimeError
(
"library has no name"
)
if
hasattr
(
value
,
'read'
):
self
.
storage
.
doc
.
save
(
value
.
read
())
else
:
self
.
storage
.
doc
.
save
(
value
)
def
hash
(
self
):
"""Returns the hexadecimal hash for the current library"""
if
not
self
.
_name
:
raise
RuntimeError
(
"library has no name"
)
return
self
.
storage
.
hash
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment