Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.cmdline
Commits
63c310c9
Commit
63c310c9
authored
Jul 03, 2020
by
Samuel GAIST
Browse files
[cmdline] Pre-commit cleanup
parent
771a2cae
Pipeline
#40957
passed with stage
in 4 minutes and 15 seconds
Changes
20
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
beat/cmdline/algorithms.py
View file @
63c310c9
...
...
@@ -34,26 +34,24 @@
###################################################################################
import
click
import
logging
import
os
import
simplejson
as
json
import
click
import
simplejson
as
json
from
beat.backend.python.algorithm
import
Storage
as
AlgorithmStorage
from
beat.core
import
algorithm
from
beat.core.execution
import
DockerExecutor
from
beat.core.dock
import
Host
from
beat.core
import
hash
from
beat.backend.python.algorithm
import
Storage
as
AlgorithmStorage
from
beat.core.dock
import
Host
from
beat.core.execution
import
DockerExecutor
from
.
import
common
from
.
import
commands
from
.decorators
import
raise_on_error
from
.
import
common
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
from
.decorators
import
raise_on_error
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/cache.py
View file @
63c310c9
...
...
@@ -34,20 +34,20 @@
###################################################################################
import
os
import
click
import
fnmatch
import
logging
import
os
import
click
import
simplejson
from
beat.core.data
import
CachedDataSource
,
load_data_index
from
beat.core.data
import
CachedDataSource
from
beat.core.data
import
load_data_index
from
beat.core.utils
import
NumpyJSONEncoder
from
.
import
common
from
.decorators
import
raise_on_error
from
.click_helper
import
AliasedGroup
from
.decorators
import
raise_on_error
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/commands.py
View file @
63c310c9
...
...
@@ -33,13 +33,13 @@
# #
###################################################################################
import
click
import
types
from
.scripts
import
main_cli
from
.decorators
import
raise_on_error
import
click
from
.
import
common
from
.decorators
import
raise_on_error
from
.scripts
import
main_cli
def
copy_func
(
f
,
name
=
None
):
...
...
beat/cmdline/common.py
View file @
63c310c9
...
...
@@ -36,29 +36,29 @@
"""Utility functions that are useful to all sub-commands"""
import
os
import
glob
import
fnmatch
import
difflib
import
collections
import
difflib
import
fnmatch
import
glob
import
logging
import
os
from
enum
import
Enum
from
enum
import
unique
import
simplejson
import
six
import
termcolor
import
simplejson
from
enum
import
Enum
,
unique
from
beat.core
import
dataformat
from
beat.core
import
algorithm
from
beat.core
import
database
from
beat.core
import
dataformat
from
beat.core
import
experiment
from
beat.core
import
library
from
beat.core
import
plotter
from
beat.core
import
plotterparameter
from
beat.core
import
protocoltemplate
from
beat.core
import
algorithm
from
beat.core
import
toolchain
from
beat.core
import
experiment
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/config.py
View file @
63c310c9
...
...
@@ -37,91 +37,85 @@
"""Configuration manipulation and display"""
import
os
import
sys
import
copy
import
logging
import
getpass
import
logging
import
os
import
sys
import
click
import
simplejson
from
.decorators
import
verbosity_option
from
.click_helper
import
AliasedGroup
from
.decorators
import
verbosity_option
logger
=
logging
.
getLogger
(
__name__
)
# Default values for the command-line utility
DEFAULTS
=
{
'platform'
:
'https://www.beat-eu.org/platform/'
,
'user'
:
getpass
.
getuser
(),
'token'
:
None
,
'prefix'
:
os
.
path
.
realpath
(
os
.
path
.
join
(
os
.
curdir
,
'prefix'
)),
'cache'
:
'cache'
,
'editor'
:
None
,
}
"""Default values for the command-line utility"""
"platform"
:
"https://www.beat-eu.org/platform/"
,
"user"
:
getpass
.
getuser
(),
"token"
:
None
,
"prefix"
:
os
.
path
.
realpath
(
os
.
path
.
join
(
os
.
curdir
,
"prefix"
)),
"cache"
:
"cache"
,
"editor"
:
None
,
}
# Documentation for the configuration parameters
DOC
=
{
'platform'
:
'Web address of the BEAT platform'
,
'user'
:
'User name for operations that create, delete or edit objects'
,
'token'
:
'Secret key of the user on the BEAT platform'
,
'prefix'
:
'Directory containing BEAT objects'
,
'cache'
:
'Directory to use for data caching (relative to prefix)'
,
'editor'
:
'Editor to be used to edit local files'
,
}
"""Documentation for configuration parameters"""
"platform"
:
"Web address of the BEAT platform"
,
"user"
:
"User name for operations that create, delete or edit objects"
,
"token"
:
"Secret key of the user on the BEAT platform"
,
"prefix"
:
"Directory containing BEAT objects"
,
"cache"
:
"Directory to use for data caching (relative to prefix)"
,
"editor"
:
"Editor to be used to edit local files"
,
}
class
Configuration
(
object
):
'''
Keeps track of configuration elements
'''
"""
Keeps track of configuration elements
"""
def
__init__
(
self
,
args
):
self
.
files
=
[
os
.
path
.
expanduser
(
'
~/.beatrc
'
),
os
.
path
.
realpath
(
'
./.beatrc
'
),
os
.
path
.
expanduser
(
"
~/.beatrc
"
),
os
.
path
.
realpath
(
"
./.beatrc
"
),
]
self
.
__data
=
copy
.
deepcopy
(
DEFAULTS
)
for
k
in
self
.
files
:
if
os
.
path
.
exists
(
k
):
with
open
(
k
,
'rt'
)
as
f
:
tmp
=
simplejson
.
load
(
f
)
with
open
(
k
,
"rt"
)
as
f
:
tmp
=
simplejson
.
load
(
f
)
self
.
__data
.
update
(
tmp
)
logger
.
info
(
"Loaded configuration file `%s'"
,
k
)
for
key
in
DEFAULTS
:
self
.
__data
[
key
]
=
args
.
get
(
'--%s'
%
key
)
or
self
.
__data
[
key
]
self
.
__data
[
key
]
=
args
.
get
(
"--%s"
%
key
)
or
self
.
__data
[
key
]
@
property
def
path
(
self
):
'''The directory for the prefix'''
return
self
.
__data
[
'prefix'
]
"""The directory for the prefix"""
return
self
.
__data
[
"prefix"
]
@
property
def
cache
(
self
):
'''The directory for the cache'''
if
os
.
path
.
isabs
(
self
.
__data
[
'cache'
]):
return
self
.
__data
[
'cache'
]
return
os
.
path
.
join
(
self
.
__data
[
'prefix'
],
self
.
__data
[
'cache'
])
"""The directory for the cache"""
if
os
.
path
.
isabs
(
self
.
__data
[
"cache"
]):
return
self
.
__data
[
"cache"
]
return
os
.
path
.
join
(
self
.
__data
[
"prefix"
],
self
.
__data
[
"cache"
])
@
property
def
database_paths
(
self
):
'''A dict of paths for databases'''
return
dict
(
(
k
,
self
.
__data
[
k
])
for
k
in
self
.
__data
if
self
.
is_database_key
(
k
)
)
"""A dict of paths for databases"""
return
dict
((
k
,
self
.
__data
[
k
])
for
k
in
self
.
__data
if
self
.
is_database_key
(
k
))
def
set
(
self
,
key
,
value
,
local
=
False
):
'''
Sets or resets a field in the configuration
'''
"""
Sets or resets a field in the configuration
"""
if
not
self
.
_is_valid_key
(
key
):
logger
.
error
(
"Don't know about parameter `%s'"
,
key
)
...
...
@@ -134,9 +128,8 @@ class Configuration(object):
self
.
save
(
local
)
def
save
(
self
,
local
=
False
):
'''
Saves contents to configuration file
"""
Saves contents to configuration file
Parameters:
...
...
@@ -144,39 +137,34 @@ class Configuration(object):
if set to ``True``, then save
configuration values to local configuration file (typically
``.beatrc``)
'''
"""
path
=
self
.
files
[
0
]
if
local
:
path
=
self
.
files
[
1
]
flags
=
os
.
O_WRONLY
|
os
.
O_CREAT
|
os
.
O_TRUNC
with
os
.
fdopen
(
os
.
open
(
path
,
flags
,
0o600
),
'
wt
'
)
as
f
:
f
.
write
(
simplejson
.
dumps
(
s
elf
.
__data
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
','
,
'
:
'
)
)
)
with
os
.
fdopen
(
os
.
open
(
path
,
flags
,
0o600
),
"
wt
"
)
as
f
:
f
.
write
(
s
implejson
.
dumps
(
self
.
__data
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
","
,
"
:
"
)
)
)
def
_is_valid_key
(
self
,
key
):
return
key
in
DEFAULTS
or
self
.
is_database_key
(
key
)
def
is_database_key
(
self
,
key
):
return
key
.
startswith
(
'database/'
)
return
key
.
startswith
(
"database/"
)
def
__str__
(
self
):
return
simplejson
.
dumps
(
self
.
__data
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
','
,
': '
)
self
.
__data
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
","
,
": "
)
)
def
as_dict
(
self
):
return
copy
.
copy
(
self
.
__data
)
def
__getattr__
(
self
,
key
):
return
self
.
__data
[
key
]
...
...
@@ -188,6 +176,7 @@ def config(ctx):
"""The manager for beat cmdline configuration."""
pass
@
config
.
command
()
@
click
.
pass_context
def
show
(
ctx
):
...
...
@@ -195,11 +184,11 @@ def show(ctx):
Lists the configuration after resolving defaults and saved variables
"""
click
.
echo
(
ctx
.
meta
[
'
config
'
])
click
.
echo
(
ctx
.
meta
[
"
config
"
])
@
config
.
command
()
@
click
.
argument
(
'
key
'
)
@
click
.
argument
(
"
key
"
)
@
click
.
pass_context
def
get
(
ctx
,
key
):
"""Prints out the contents of a single field.
...
...
@@ -221,19 +210,22 @@ def get(ctx, key):
-----
* If the key is not found.
"""
value
=
getattr
(
ctx
.
meta
[
'
config
'
],
key
)
value
=
getattr
(
ctx
.
meta
[
"
config
"
],
key
)
if
value
is
None
:
# Exit the command line with ClickException in case of errors.
raise
click
.
ClickException
(
"The requested key `{}' does not exist"
.
format
(
key
))
raise
click
.
ClickException
(
"The requested key `{}' does not exist"
.
format
(
key
))
click
.
echo
(
value
)
@
config
.
command
()
@
click
.
argument
(
'args'
,
nargs
=-
1
)
@
click
.
option
(
'--local/--not-local'
,
default
=
False
,
help
=
'Save values on the '
'local configuration file (.beatrc) instead of using the global '
'file (~/.beatrc)'
)
@
click
.
argument
(
"args"
,
nargs
=-
1
)
@
click
.
option
(
"--local/--not-local"
,
default
=
False
,
help
=
"Save values on the "
"local configuration file (.beatrc) instead of using the global "
"file (~/.beatrc)"
,
)
@
click
.
pass_context
def
set
(
ctx
,
args
,
local
):
"""Sets the value for a key.
...
...
@@ -260,10 +252,10 @@ def set(ctx, args, local):
"""
if
len
(
args
)
%
2
!=
0
:
raise
click
.
BadParameter
(
'
You must provide pair(s) of key/value
'
)
raise
click
.
BadParameter
(
"
You must provide pair(s) of key/value
"
)
try
:
for
idx
in
range
(
0
,
len
(
args
),
2
):
ctx
.
meta
[
'
config
'
].
set
(
args
[
idx
],
args
[
idx
+
1
],
local
)
ctx
.
meta
[
"
config
"
].
set
(
args
[
idx
],
args
[
idx
+
1
],
local
)
except
Exception
:
raise
click
.
ClickException
(
"Failed to change the configuration."
)
beat/cmdline/databases.py
View file @
63c310c9
...
...
@@ -34,31 +34,30 @@
###################################################################################
import
os
import
click
import
glob
import
random
import
zmq
import
logging
import
os
import
random
import
click
import
simplejson
import
zmq
from
beat.core.hash
import
toPath
from
beat.core.hash
import
hashDataset
from
beat.core.utils
import
NumpyJSONEncoder
from
beat.core.database
import
Database
from
beat.core.data
import
RemoteDataSource
from
beat.core
import
dock
from
beat.core
import
inputs
from
beat.core
import
utils
from
beat.core.data
import
RemoteDataSource
from
beat.core.database
import
Database
from
beat.core.hash
import
hashDataset
from
beat.core.hash
import
toPath
from
beat.core.utils
import
NumpyJSONEncoder
from
.
import
common
from
.
import
commands
from
.decorators
import
raise_on_error
from
.
import
common
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
from
.decorators
import
raise_on_error
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/dataformats.py
View file @
63c310c9
...
...
@@ -35,17 +35,17 @@
import
logging
import
click
from
beat.core
import
dataformat
from
.
import
common
from
.
import
commands
from
.decorators
import
raise_on_error
from
.
import
common
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
from
.decorators
import
raise_on_error
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/decorators.py
View file @
63c310c9
...
...
@@ -34,9 +34,11 @@
import
logging
import
click
from
functools
import
wraps
import
click
from
.log
import
set_verbosity_level
# This needs to be beat so that logger is configured for all beat packages.
...
...
beat/cmdline/environments.py
View file @
63c310c9
...
...
@@ -34,17 +34,16 @@
###################################################################################
import
click
import
logging
import
click
import
simplejson
as
json
from
beat.core.dock
import
Host
from
.
import
common
from
.click_helper
import
AliasedGroup
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/libraries.py
View file @
63c310c9
...
...
@@ -74,18 +74,17 @@ Options:
"""
import
logging
import
click
from
beat.core
import
library
from
.
import
common
from
.
import
commands
from
.decorators
import
raise_on_error
from
.
import
common
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
from
.decorators
import
raise_on_error
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/log.py
View file @
63c310c9
...
...
@@ -5,6 +5,7 @@
import
logging
def
set_verbosity_level
(
logger
,
level
):
"""Sets the log level for the given logger.
...
...
@@ -30,7 +31,7 @@ def set_verbosity_level(logger, level):
0
:
logging
.
ERROR
,
1
:
logging
.
WARNING
,
2
:
logging
.
INFO
,
3
:
logging
.
DEBUG
3
:
logging
.
DEBUG
,
}[
level
]
# set this log level to the logger with the specified name
...
...
beat/cmdline/plotterparameters.py
View file @
63c310c9
...
...
@@ -36,13 +36,12 @@
import
click
from
.
import
common
from
.
import
commands
from
.decorators
import
raise_on_error
from
.
import
common
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
from
.decorators
import
raise_on_error
def
pull_impl
(
webapi
,
prefix
,
names
,
force
,
indentation
,
format_cache
):
...
...
beat/cmdline/plotters.py
View file @
63c310c9
...
...
@@ -34,31 +34,30 @@
###################################################################################
import
collections
import
io
import
logging
import
os
import
sys
import
click
import
logging
import
simplejson
import
collections
from
PIL
import
Image
from
beat.core
import
plotter
from
beat.core
import
dataformat
from
beat.core
import
plotter
from
beat.core.plotterparameter
import
Storage
as
PPStorage
from
.
import
common
from
.
import
commands
from
.dataformats
import
pull_impl
as
dataformats_pull
from
.plotterparameters
import
pull_impl
as
plotterparameters_pull
from
.libraries
import
pull_impl
as
libraries_pull
from
.decorators
import
raise_on_error
from
.
import
common
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
from
.dataformats
import
pull_impl
as
dataformats_pull
from
.decorators
import
raise_on_error
from
.libraries
import
pull_impl
as
libraries_pull
from
.plotterparameters
import
pull_impl
as
plotterparameters_pull
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/protocoltemplates.py
View file @
63c310c9
...
...
@@ -34,16 +34,15 @@
###################################################################################
import
click
import
logging
from
.
import
c
ommands
import
c
lick
from
.
import
commands
from
.click_helper
import
AliasedGroup
from
.click_helper
import
AssetCommand
from
.click_helper
import
AssetInfo
logger
=
logging
.
getLogger
(
__name__
)
...
...
beat/cmdline/scripts/__init__.py
View file @
63c310c9
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
beat/cmdline/scripts/main_cli.py
View file @
63c310c9
...
...
@@ -2,12 +2,15 @@
"""
import
logging
import
pkg_resources
import
click
import
pkg_resources
from
click_plugins
import
with_plugins
from
..click_helper
import
AliasedGroup
from
..decorators
import
verbosity_option
from
..config
import
Configuration
from
..decorators
import
verbosity_option
# defines our own logging level for extra information to be printed
logging
.
EXTRA
=
15
...
...
beat/cmdline/status.py
View file @
63c310c9
...
...
@@ -51,10 +51,10 @@ from .decorators import raise_on_error
@
raise_on_error
def
status
(
ctx
):
"""Shows the working folder status for different objects"""
path
=
ctx
.
meta
[
'
config
'
].
path
with
common
.
make_webapi
(
ctx
.
meta
[
'
config
'
])
as
webapi
:
retval
=
common
.
status
(
webapi
,
path
,
'
dataformat
'
)[
0
]
retval
+=
common
.
status
(
webapi
,
path
,
'
algorithm
'
)[
0
]
retval
+=
common
.
status
(
webapi
,
path
,
'
toolchain
'
)[
0
]