Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.backend.python
Commits
aa4eff11
Commit
aa4eff11
authored
Mar 17, 2017
by
Philip ABBET
Browse files
More refactoring
parent
81392e5a
Changes
4
Hide whitespace changes
Inline
Side-by-side
beat/backend/python/database.py
View file @
aa4eff11
...
...
@@ -30,17 +30,40 @@
import
os
import
sys
# import collections
import
six
import
simplejson
from
.
import
loader
from
.
import
utils
from
.dataformat
import
DataFormat
class
Storage
(
utils
.
CodeStorage
):
"""Resolves paths for databases
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
"""
def
__init__
(
self
,
prefix
,
name
):
if
name
.
count
(
'/'
)
!=
1
:
raise
RuntimeError
(
"invalid database name: `%s'"
%
name
)
self
.
name
,
self
.
version
=
name
.
split
(
'/'
)
self
.
fullname
=
name
path
=
os
.
path
.
join
(
prefix
,
'databases'
,
name
)
super
(
Storage
,
self
).
__init__
(
path
,
'python'
)
#views are coded in Python
# from . import dataformat
# from . import hash
# from . import utils
# from . import prototypes
class
View
(
object
):
...
...
@@ -207,7 +230,9 @@ class Database(object):
self
.
_name
=
None
self
.
prefix
=
prefix
self
.
dataformats
=
{}
# preloaded dataformats
self
.
storage
=
None
self
.
errors
=
[]
self
.
data
=
None
# if the user has not provided a cache, still use one for performance
...
...
@@ -220,8 +245,31 @@ class Database(object):
"""Loads the database"""
self
.
_name
=
data
json_path
=
os
.
path
.
join
(
prefix
,
'databases'
,
name
+
'.json'
)
with
open
(
json_path
,
'rb'
)
as
f
:
self
.
data
=
simplejson
.
load
(
f
)
self
.
storage
=
Storage
(
self
.
prefix
,
self
.
_name
)
json_path
=
self
.
storage
.
json
.
path
if
not
self
.
storage
.
json
.
exists
():
self
.
errors
.
append
(
'Database declaration file not found: %s'
%
json_path
)
return
with
open
(
json_path
,
'rb'
)
as
f
:
self
.
data
=
simplejson
.
load
(
f
)
for
protocol
in
self
.
data
[
'protocols'
]:
for
_set
in
protocol
[
'sets'
]:
for
key
,
value
in
_set
[
'outputs'
].
items
():
if
value
in
self
.
dataformats
:
continue
if
value
in
dataformat_cache
:
dataformat
=
dataformat_cache
[
value
]
else
:
dataformat
=
DataFormat
(
self
.
prefix
,
value
)
dataformat_cache
[
value
]
=
dataformat
self
.
dataformats
[
value
]
=
dataformat
@
property
...
...
@@ -237,6 +285,11 @@ class Database(object):
return
self
.
data
.
get
(
'schema_version'
,
1
)
@
property
def
valid
(
self
):
return
not
bool
(
self
.
errors
)
@
property
def
protocols
(
self
):
"""The declaration of all the protocols of the database"""
...
...
beat/backend/python/dbexecution.py
View file @
aa4eff11
...
...
@@ -119,10 +119,10 @@ class DBExecutor(object):
database_cache
=
database_cache
if
database_cache
is
not
None
else
{}
self
.
dataformat_cache
=
dataformat_cache
if
dataformat_cache
is
not
None
else
{}
self
.
_load
(
data
,
self
.
dataformat_cache
,
database_cache
)
self
.
_load
(
data
,
database_cache
)
def
_load
(
self
,
data
,
dataformat_cache
,
database_cache
):
def
_load
(
self
,
data
,
database_cache
):
"""Loads the block execution information"""
# reset
...
...
beat/backend/python/hash.py
View file @
aa4eff11
...
...
@@ -32,6 +32,71 @@
import
hashlib
def
_sha256
(
s
):
"""A python2/3 replacement for :py:func:`haslib.sha256`"""
try
:
if
isinstance
(
s
,
str
):
s
=
six
.
u
(
s
)
return
hashlib
.
sha256
(
s
.
encode
(
'utf8'
)).
hexdigest
()
except
:
return
hashlib
.
sha256
(
s
).
hexdigest
()
def
_stringify
(
dictionary
):
names
=
sorted
(
dictionary
.
keys
())
converted_dictionary
=
'{'
for
name
in
names
:
converted_dictionary
+=
'"%s":%s,'
%
(
name
,
str
(
dictionary
[
name
]))
if
len
(
converted_dictionary
)
>
1
:
converted_dictionary
=
converted_dictionary
[:
-
1
]
converted_dictionary
+=
'}'
return
converted_dictionary
def
hash
(
dictionary_or_string
):
if
isinstance
(
dictionary_or_string
,
dict
):
return
_sha256
(
_stringify
(
dictionary_or_string
))
else
:
return
_sha256
(
dictionary_or_string
)
def
hashJSON
(
contents
,
description
):
"""Hashes the pre-loaded JSON object using :py:func:`hashlib.sha256`
Excludes description changes
"""
if
description
in
contents
:
contents
=
copy
.
deepcopy
(
contents
)
#temporary copy
del
contents
[
description
]
contents
=
simplejson
.
dumps
(
contents
,
sort_keys
=
True
)
return
hashlib
.
sha256
(
contents
).
hexdigest
()
def
hashJSONFile
(
path
,
description
):
"""Hashes the JSON file contents using :py:func:`hashlib.sha256`
Excludes description changes
"""
try
:
with
open
(
path
,
'rb'
)
as
f
:
return
hashJSON
(
simplejson
.
load
(
f
,
object_pairs_hook
=
collections
.
OrderedDict
),
description
)
#preserve order
except
simplejson
.
JSONDecodeError
:
# falls back to normal file content hashing
return
hashFileContents
(
path
)
def
hashFileContents
(
path
):
"""Hashes the file contents using :py:func:`hashlib.sha256`."""
...
...
beat/backend/python/utils.py
0 → 100644
View file @
aa4eff11
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.core module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import
os
import
shutil
import
collections
import
simplejson
import
six
from
.
import
hash
def
safe_rmfile
(
f
):
"""Safely removes a file from the disk"""
if
os
.
path
.
exists
(
f
):
os
.
unlink
(
f
)
def
safe_rmdir
(
f
):
"""Safely removes the directory containg a given file from the disk"""
d
=
os
.
path
.
dirname
(
f
)
if
not
os
.
path
.
exists
(
d
):
return
if
not
os
.
listdir
(
d
):
os
.
rmdir
(
d
)
def
extension_for_language
(
language
):
"""Returns the preferred extension for a given programming language
The set of languages supported must match those declared in our
``common.json`` schema.
Parameters:
language (str) The language for which you'd like to get the extension for.
Returns:
str: The extension for the given language, including a leading ``.`` (dot)
Raises:
KeyError: If the language is not defined in our internal dictionary.
"""
return
dict
(
unknown
=
''
,
cxx
=
'.so'
,
matlab
=
'.m'
,
python
=
'.py'
,
r
=
'.r'
,
)[
language
]
class
File
(
object
):
"""User helper to read and write file objects"""
def
__init__
(
self
,
path
,
binary
=
False
):
self
.
path
=
path
self
.
binary
=
binary
def
exists
(
self
):
return
os
.
path
.
exists
(
self
.
path
)
def
load
(
self
):
mode
=
'rb'
if
self
.
binary
else
'rt'
with
open
(
self
.
path
,
mode
)
as
f
:
return
f
.
read
()
def
try_load
(
self
):
if
os
.
path
.
exists
(
self
.
path
):
return
self
.
load
()
return
None
def
backup
(
self
):
if
not
os
.
path
.
exists
(
self
.
path
):
return
#no point in backing-up
backup
=
self
.
path
+
'~'
if
os
.
path
.
exists
(
backup
):
os
.
remove
(
backup
)
shutil
.
copy
(
self
.
path
,
backup
)
def
save
(
self
,
contents
):
d
=
os
.
path
.
dirname
(
self
.
path
)
if
not
os
.
path
.
exists
(
d
):
os
.
makedirs
(
d
)
if
os
.
path
.
exists
(
self
.
path
):
self
.
backup
()
mode
=
'wb'
if
self
.
binary
else
'wt'
with
open
(
self
.
path
,
mode
)
as
f
:
f
.
write
(
contents
)
def
remove
(
self
):
safe_rmfile
(
self
.
path
)
safe_rmfile
(
self
.
path
+
'~'
)
#backup
safe_rmdir
(
self
.
path
)
#remove containing directory
class
Storage
(
object
):
"""Resolves paths for objects that provide only a description
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
"""
def
__init__
(
self
,
path
):
self
.
path
=
path
self
.
json
=
File
(
self
.
path
+
'.json'
)
self
.
doc
=
File
(
self
.
path
+
'.rst'
)
def
hash
(
self
,
description
=
'description'
):
"""The 64-character hash of the database declaration JSON"""
return
hash
.
hashJSONFile
(
self
.
json
.
path
,
description
)
def
exists
(
self
):
"""If the database declaration file exists"""
return
self
.
json
.
exists
()
def
load
(
self
):
"""Loads the JSON declaration as a file"""
tp
=
collections
.
namedtuple
(
'Storage'
,
[
'declaration'
,
'description'
])
return
tp
(
self
.
json
.
load
(),
self
.
doc
.
try_load
())
def
save
(
self
,
declaration
,
description
=
None
):
"""Saves the JSON declaration as files"""
if
description
:
self
.
doc
.
save
(
description
.
encode
(
'utf8'
))
if
not
isinstance
(
declaration
,
six
.
string_types
):
declaration
=
simplejson
.
dumps
(
declaration
,
indent
=
4
)
self
.
json
.
save
(
declaration
)
def
remove
(
self
):
"""Removes the object from the disk"""
self
.
json
.
remove
()
self
.
doc
.
remove
()
class
CodeStorage
(
object
):
"""Resolves paths for objects that provide a description and code
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the database object in the format
``<name>/<version>``.
language (str): One of the valdid programming languages
"""
def
__init__
(
self
,
path
,
language
=
None
):
self
.
path
=
path
self
.
json
=
File
(
self
.
path
+
'.json'
)
self
.
doc
=
File
(
self
.
path
+
'.rst'
)
self
.
_language
=
language
or
self
.
__auto_discover_language
()
self
.
code
=
File
(
self
.
path
+
\
extension_for_language
(
self
.
_language
),
binary
=
True
)
def
__auto_discover_language
(
self
,
json
=
None
):
"""Discovers and sets the language from its own JSON descriptor"""
try
:
text
=
json
or
self
.
json
.
load
()
json
=
simplejson
.
loads
(
text
)
return
json
[
'language'
]
except
IOError
:
return
'unknown'
@
property
def
language
(
self
):
return
self
.
_language
@
language
.
setter
def
language
(
self
,
value
):
self
.
_language
=
value
self
.
code
=
File
(
self
.
path
+
extension_for_language
(
self
.
_language
),
binary
=
True
)
def
hash
(
self
):
"""The 64-character hash of the database declaration JSON"""
if
self
.
code
.
exists
():
return
hash
.
hash
(
dict
(
json
=
hash
.
hashJSONFile
(
self
.
json
.
path
,
'description'
),
code
=
hash
.
hashFileContents
(
self
.
code
.
path
),
))
else
:
return
hash
.
hash
(
dict
(
json
=
hash
.
hashJSONFile
(
self
.
json
.
path
,
'description'
),
))
def
exists
(
self
):
"""If the database declaration file exists"""
return
self
.
json
.
exists
()
and
self
.
code
.
exists
()
def
load
(
self
):
"""Loads the JSON declaration as a file"""
tp
=
collections
.
namedtuple
(
'CodeStorage'
,
[
'declaration'
,
'code'
,
'description'
])
return
tp
(
self
.
json
.
load
(),
self
.
code
.
try_load
(),
self
.
doc
.
try_load
())
def
save
(
self
,
declaration
,
code
=
None
,
description
=
None
):
"""Saves the JSON declaration and the code as files"""
super
(
CodeStorage
,
self
).
save
(
declaration
,
description
)
if
code
:
if
self
.
_language
==
'unknown'
:
self
.
language
=
self
.
__auto_discover_language
(
declaration
)
self
.
code
.
save
(
code
)
def
remove
(
self
):
"""Removes the object from the disk"""
super
(
CodeStorage
,
self
).
remove
()
self
.
code
.
remove
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment