Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.cmdline
Commits
4eb87491
Commit
4eb87491
authored
May 01, 2018
by
André Anjos
💬
Browse files
Closes
#24
parent
d167e6e7
Changes
11
Hide whitespace changes
Inline
Side-by-side
beat/cmdline/config.py
View file @
4eb87491
...
...
@@ -29,50 +29,40 @@
"""Configuration manipulation and display
Usage:
%(prog)s config save
%(prog)s config list
%(prog)s config set (<key> <value>)...
%(prog)s config show
%(prog)s config set [--local] (<key> <value>)...
%(prog)s config get <key>...
%(prog)s config color [--list] [<theme>]
%(prog)s config --help
Commands:
save Saves the commandline configuration to your prefix
list Lists the configuration after resolving defaults and saved variables
set Sets a specific known field to a value (or reset it, if no value given)
show Lists the configuration after resolving defaults and saved variables
set Sets a specific known field to a value
get Prints out the contents of a single field
color Sets the color theme (or reset it, if a theme is not given)
Options:
-h, --help Display this screen
-l, --list Lists all possible color themes
-h, --help Display this screen
-l, --local Save values on the local configuration file (.beatrc) instead
of using the global file (~/.beatrc)
Examples:
To
list all available
configuration parameters:
To
show currently set
configuration parameters
and defaults
:
$ %(prog)s config
list
$ %(prog)s config
show
To save a different user name token to a file and save results locally
(notice you can pass multiple parameters at once using key-value pairs):
To save a different user name token to a file and save results locally - i.e.
doesn't override the global configuration file (notice you can pass multiple
parameters at once using key-value pairs):
$ %(prog)s config set user "me"
token "1234567890abcdef1234567890abcde"
$ %(prog)s config set
--local
user "me"
prefix `pwd`/prefix
To query for a specific parameter:
$ %(prog)s config get token
1234567890abcdef1234567890abcde
To change the color schemes to a ``dark`` theme and save results locally:
$ %(prog)s config color dark
To list all possible color schemes:
$ %(prog)s config color --list
"""
...
...
@@ -83,75 +73,27 @@ import logging
logger
=
logging
.
getLogger
(
__name__
)
import
simplejson
import
termcolor
import
getpass
DEFAULTS
=
{
'platform'
:
'https://www.beat-eu.org/platform/'
,
'user'
:
None
,
'user'
:
getpass
.
getuser
()
,
'token'
:
None
,
'cache'
:
'cache'
,
#relative path to <prefix>
}
COLOR_THEMES
=
{
'none'
:
{
'color_critical'
:
''
,
'color_error'
:
''
,
'color_warn'
:
''
,
'color_info'
:
''
,
'color_debug'
:
''
,
},
'dark'
:
{
'color_critical'
:
'bold_red'
,
'color_error'
:
'fg_red'
,
'color_warn'
:
'bold_yellow'
,
'color_info'
:
'fg_green'
,
'color_debug'
:
'fg_white'
,
},
'light'
:
{
'color_critical'
:
'bold_red'
,
'color_error'
:
'fg_red'
,
'color_warn'
:
'bold_blue'
,
'color_info'
:
'fg_green'
,
'color_debug'
:
'fg_grey'
,
},
}
DEFAULTS
.
update
(
COLOR_THEMES
[
'none'
])
'prefix'
:
os
.
path
.
realpath
(
os
.
path
.
join
(
os
.
curdir
,
'prefix'
)),
'cache'
:
'cache'
,
}
"""Default values for the command-line utility"""
DOC
=
{
'platform'
:
'Web address of the BEAT platform'
,
'user'
:
'User name for operations that create, delete or edit objects'
,
'token'
:
'Secret key of the user on the BEAT platform'
,
'cache'
:
'Path to the directory to use for data caching'
,
'color_critical'
:
'Color of critical messages in the terminal'
,
'color_error'
:
'Color of error messages in the terminal'
,
'color_warn'
:
'Color of warning messages in the terminal'
,
'color_info'
:
'Color of informational messages in the terminal'
,
'color_debug'
:
'Color of debug messages in the terminal'
,
}
def
colorlog_to_termcolor
(
color
):
'''Returns the termcolor configuration tuple (color, on_color, attrs)'''
info
=
[
t
for
t
in
color
.
split
(
','
)
if
t
.
strip
()]
color
=
None
on_color
=
None
attrs
=
[]
for
i
in
info
:
c
=
i
.
split
(
'_'
)
if
c
[
0
]
in
(
'bold'
,
'fg'
):
# text color
color
=
c
[
1
]
if
c
[
0
]
==
'bold'
:
attrs
.
append
(
'bold'
)
if
c
[
0
]
in
(
'bg'
):
# background color
on_color
=
'on_%s'
%
c
[
1
]
return
color
,
on_color
,
attrs
'prefix'
:
'Directory containing BEAT objects'
,
'cache'
:
'Directory to use for data caching (relative to prefix)'
,
}
"""Documentation for configuration parameters"""
class
Configuration
(
object
):
...
...
@@ -159,56 +101,38 @@ class Configuration(object):
def
__init__
(
self
,
args
):
self
.
path
=
args
.
get
(
'--prefix'
,
'.'
)
self
.
load
()
#loads defaults and local configuration, if one exists
#overwrites preferences, if the user has set any
self
.
__data
[
'cache'
]
=
args
.
get
(
'--cache'
)
or
self
.
__data
[
'cache'
]
self
.
__data
[
'platform'
]
=
args
.
get
(
'--platform'
)
or
self
.
__data
[
'platform'
]
self
.
__data
[
'user'
]
=
args
.
get
(
'--user'
)
or
self
.
__data
[
'user'
]
self
.
__data
[
'token'
]
=
args
.
get
(
'--token'
)
or
self
.
__data
[
'token'
]
if
args
.
get
(
'--color'
):
if
args
[
'--color'
]
not
in
COLOR_THEMES
:
print
(
"ERROR: color theme must be one of `%s' - `%s' is invalid"
%
\
(
','
.
join
(
COLOR_THEMES
.
keys
()),
args
[
'--color'
]))
sys
.
exit
(
1
)
self
.
__data
.
update
(
COLOR_THEMES
[
args
[
'--color'
]])
def
__config_file
(
self
):
return
os
.
path
.
join
(
self
.
path
,
'.beat'
,
'config.json'
)
from
bob.extension.config
import
load
,
mod_to_context
def
load
(
self
):
'''Loads contents from file'''
self
.
files
=
[
os
.
path
.
expanduser
(
'~/.beatrc'
),
os
.
path
.
realpath
(
'./.beatrc'
),
]
self
.
__data
=
copy
.
deepcopy
(
DEFAULTS
)
for
k
in
self
.
files
:
if
os
.
path
.
exists
(
k
):
with
open
(
k
,
'rt'
)
as
f
:
tmp
=
simplejson
.
load
(
f
)
self
.
__data
.
update
(
tmp
)
logger
.
info
(
"Loaded configuration file `%s'"
,
k
)
try
:
for
key
in
DEFAULTS
:
self
.
__data
[
key
]
=
args
.
get
(
'--%s'
%
key
)
or
self
.
__data
[
key
]
c
=
self
.
__config_file
()
if
not
os
.
path
.
exists
(
c
):
return
with
open
(
c
,
'rt'
)
as
f
:
user_data
=
simplejson
.
load
(
f
)
for
k
in
user_data
:
if
self
.
_is_valid_key
(
k
):
self
.
__data
[
k
]
=
user_data
[
k
]
except
simplejson
.
JSONDecodeError
:
raise
# print("WARNING: invalid state file at `%s' - removing and " \
# "re-starting..." % c)
# from beat.core.utils import safe_rmfile
# safe_rmfile(c)
@
property
def
path
(
self
):
'''The directory for the prefix'''
return
self
.
__data
[
'prefix'
]
@
property
def
cache
(
self
):
'''The directory for the cache'''
if
not
os
.
path
.
isabs
(
self
.
__data
[
'cache'
]):
return
os
.
path
.
join
(
self
.
path
,
self
.
__data
[
'cache'
]
)
return
self
.
__data
[
'cache'
]
if
os
.
path
.
isabs
(
self
.
__data
[
'cache'
]):
return
self
.
__data
[
'cache'
]
return
os
.
path
.
join
(
self
.
__data
[
'prefix'
],
self
.
__data
[
'cache'
]
)
@
property
...
...
@@ -218,11 +142,11 @@ class Configuration(object):
return
dict
((
k
,
self
.
__data
[
k
])
for
k
in
self
.
__data
if
self
.
is_database_key
(
k
))
def
set
(
self
,
key
,
value
):
def
set
(
self
,
key
,
value
,
local
=
False
):
'''Sets or resets a field in the configuration'''
if
not
self
.
_is_valid_key
(
key
):
print
(
"ERROR: d
on't know about parameter `%s'"
%
key
)
logger
.
error
(
"D
on't know about parameter `%s'"
,
key
)
sys
.
exit
(
1
)
if
value
is
not
None
:
...
...
@@ -230,64 +154,39 @@ class Configuration(object):
elif
key
in
DEFAULTS
:
self
.
__data
[
key
]
=
DEFAULTS
[
key
]
self
.
save
()
self
.
save
(
local
)
def
color
(
self
,
v
al
u
e
):
'''S
ets the color theme'''
def
save
(
self
,
local
=
F
al
s
e
):
'''S
aves contents to configuration file
if
value
is
None
:
value
=
'none'
Parameters:
elif
value
not
in
COLOR_THEMES
:
print
(
"ERROR: color theme must be one of `%s' - `%s' is invalid"
%
\
(
','
.
join
(
COLOR_THEMES
.
keys
()),
value
))
sys
.
exit
(
1
)
local (:py:class:`bool`, Optional): if set to ``True``, then save
configuration values to local configuration file (typically ``.beatrc``)
self
.
__data
.
update
(
COLOR_THEMES
[
value
])
self
.
save
()
'''
path
=
self
.
files
[
0
]
if
local
:
path
=
self
.
files
[
1
]
def
save
(
self
):
'''Saves contents to file'''
with
os
.
fdopen
(
os
.
open
(
path
,
os
.
O_WRONLY
|
os
.
O_CREAT
,
0o600
),
'wt'
)
as
f
:
f
.
write
(
simplejson
.
dumps
(
self
.
__data
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
','
,
': '
)))
c
=
self
.
__config_file
()
dirname
=
os
.
path
.
dirname
(
c
)
if
not
os
.
path
.
exists
(
dirname
):
os
.
makedirs
(
dirname
)
with
os
.
fdopen
(
os
.
open
(
c
,
os
.
O_WRONLY
|
os
.
O_CREAT
,
0o600
),
'wt'
)
as
f
:
simplejson
.
dump
(
self
.
__data
,
f
,
indent
=
4
)
def
_is_valid_key
(
self
,
key
):
return
key
in
DEFAULTS
or
self
.
is_database_key
(
key
)
def
is_database_key
(
self
,
key
):
return
key
.
startswith
(
'database/'
)
def
__str__
(
self
):
retval
=
[]
c
=
self
.
__config_file
()
if
os
.
path
.
exists
(
c
):
retval
.
append
(
"-> loaded configuration file `%s'"
%
c
)
else
:
retval
.
append
(
"-> configuration file `%s' does *not* exist"
%
c
)
for
key
in
sorted
([
k
for
k
in
DOC
if
not
k
.
startswith
(
'color'
)]):
value
=
self
.
__data
[
key
]
value
=
"`%s'"
%
value
if
value
is
not
None
else
'<unset>'
retval
.
append
(
" * %-15s: %s"
%
(
key
,
value
))
for
key
in
sorted
([
k
for
k
in
self
.
__data
if
self
.
is_database_key
(
k
)]):
value
=
self
.
__data
[
key
]
value
=
"`%s'"
%
value
if
value
is
not
None
else
'<unset>'
retval
.
append
(
" * %-15s: %s"
%
(
key
,
value
))
for
key
in
sorted
([
k
for
k
in
DOC
if
k
.
startswith
(
'color'
)]):
value
=
self
.
__data
[
key
]
color
,
on_color
,
attrs
=
colorlog_to_termcolor
(
value
)
example
=
termcolor
.
colored
(
'example'
,
color
,
on_color
,
attrs
)
retval
.
append
(
" * %-15s: `%s' (%s)"
%
(
key
,
value
,
example
))
def
__str__
(
self
):
return
'
\n
'
.
join
(
retval
)
return
simplejson
.
dumps
(
self
.
__data
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
','
,
': '
))
def
__getattr__
(
self
,
key
):
...
...
@@ -297,39 +196,19 @@ class Configuration(object):
def
process
(
args
):
# Must display the list of options?
if
args
[
'save'
]:
args
[
'config'
].
save
()
return
0
elif
args
[
'list'
]:
if
args
[
'show'
]:
print
(
args
[
'config'
])
return
0
elif
args
[
'set'
]:
for
k
,
v
in
zip
(
args
[
'<key>'
],
args
[
'<value>'
]):
args
[
'config'
].
set
(
k
,
v
)
args
[
'config'
].
set
(
k
,
v
,
args
[
'--local'
]
)
return
0
elif
args
[
'get'
]:
for
k
in
args
[
'<key>'
]:
print
(
getattr
(
args
[
'config'
],
k
))
return
0
elif
args
[
'color'
]:
if
args
[
'--list'
]:
for
k
in
COLOR_THEMES
:
print
(
"`%s' theme:"
%
k
)
for
par
in
COLOR_THEMES
[
k
]:
value
=
COLOR_THEMES
[
k
][
par
]
color
,
on_color
,
attrs
=
colorlog_to_termcolor
(
value
)
example
=
termcolor
.
colored
(
'example'
,
color
,
on_color
,
attrs
)
print
(
" * %-15s: `%s' (%s)"
%
(
par
,
value
,
example
))
return
0
args
[
'config'
].
color
(
args
[
'<theme>'
])
return
0
# Should not happen
logger
.
error
(
"
u
nrecognized `config' subcommand"
)
logger
.
error
(
"
U
nrecognized `config' subcommand"
)
return
1
beat/cmdline/scripts/beat.py
View file @
4eb87491
...
...
@@ -30,7 +30,7 @@
Usage:
%(prog)s [--verbose ...] [--prefix=<path>] [--cache=<path>] [--user=<user>]
[--platform=<url>] [--token=<token>]
[--color=<theme>]
[--platform=<url>] [--token=<token>]
[--test-mode] <command> [<args>...]
%(prog)s (--help | -h)
%(prog)s (--version | -V)
...
...
@@ -40,14 +40,20 @@ Options:
-h, --help Show this screen
-v, --verbose Increases the verbosity (may appear multiple times)
-V, --version Show version
-p, --prefix=<path> Prefix of your local data [default: .]
-c, --cache=<path> Cache prefix, otherwise defaults to `<prefix>/cache'
-o, --color=<theme> The color theme to use for colored messages (choose
between `dark', for dark backgrounds or `light', for
lighther ones - the default is `none')
-t, --token=<token> User token for server operations
-u, --user=<user> The user name on the remote platform
-p, --prefix=<path> Overrides the prefix of your local data. If not set use
the value from your RC file
[default: %(prefix)s]
-c, --cache=<path> Overrides the cache prefix. If not set, use the value
from your RC file, otherwise defaults to
`<prefix>/%(cache)s'
-t, --token=<token> Overrides the user token for server operations. If not
set, use the value from your RC file. There are no
defaults for this option.
-u, --user=<user> Overrides the user name on the remote platform. If not
set, use the value from your RC file.
[default: %(user)s]
-m, --platform=<url> The URL of the BEAT platform to access
[default: %(platform)s]
-T, --test-mode Assume test mode and doesn't setup the logging module
...
...
@@ -65,6 +71,7 @@ Commands:
See 'beat <command> --help' for more information on a specific command.
N.B.: The values of options "prefix", "cache" "user", "token", "platform" and
"""
import
os
...
...
@@ -76,7 +83,6 @@ from ..config import Configuration
import
difflib
import
logging
import
colorlog
# defines our own logging level for extra information to be printed
...
...
@@ -98,17 +104,21 @@ def main(user_input=None):
else
:
arguments
=
sys
.
argv
[
1
:]
from
..config
import
DEFAULTS
prog
=
os
.
path
.
basename
(
sys
.
argv
[
0
])
completions
=
dict
(
prog
=
prog
,
version
=
__version__
,
)
**
DEFAULTS
,
)
args
=
docopt
(
__doc__
%
completions
,
argv
=
arguments
,
options_first
=
True
,
version
=
'BEAT command-line swiss army knife v%s'
%
__version__
,
)
)
try
:
...
...
@@ -150,23 +160,12 @@ def main(user_input=None):
console_handler
=
logging
.
StreamHandler
()
console_handler
.
setLevel
(
logging
.
INFO
)
# default level
format_str
=
"%(
log_color)s%(
message)s"
format_str
=
"%(message)s"
if
args
[
'--verbose'
]
>=
2
:
format_str
=
"%(log_color)s[%(asctime)s - %(name)s]%(reset)s %(levelname)s: %(message)s"
# Some coloring
formatter
=
colorlog
.
ColoredFormatter
(
format_str
,
log_colors
=
{
'DEBUG'
:
config
.
color_debug
,
'EXTRA'
:
config
.
color_info
,
'INFO'
:
config
.
color_info
,
'WARNING'
:
config
.
color_warn
,
'ERROR'
:
config
.
color_error
,
'CRITICAL'
:
config
.
color_critical
,
},
style
=
'%'
,
datefmt
=
"%d/%b/%Y %H:%M:%S"
)
format_str
=
"[%(asctime)s - %(name)s]%(reset)s %(levelname)s: %(message)s"
formatter
=
logging
.
Formatter
(
format_str
,
style
=
'%'
,
datefmt
=
"%d/%b/%Y %H:%M:%S"
)
console_handler
.
setFormatter
(
formatter
)
logger
.
addHandler
(
console_handler
)
...
...
beat/cmdline/test/__init__.py
View file @
4eb87491
...
...
@@ -33,6 +33,7 @@ import tempfile
import
shutil
import
subprocess
import
pkg_resources
import
contextlib
import
six.moves.urllib
as
urllib
...
...
@@ -95,3 +96,14 @@ def setup_package():
def
teardown_package
():
shutil
.
rmtree
(
prefix_folder
)
@
contextlib
.
contextmanager
def
temp_cwd
():
tempdir
=
tempfile
.
mkdtemp
(
prefix
=
__name__
,
suffix
=
'.cwd'
)
curdir
=
os
.
getcwd
()
os
.
chdir
(
tempdir
)
try
:
yield
tempdir
finally
:
os
.
chdir
(
curdir
)
shutil
.
rmtree
(
tempdir
)
beat/cmdline/test/test_algorithms.py
View file @
4eb87491
...
...
@@ -33,7 +33,7 @@ import os
import
shutil
import
json
from
.
import
platform
,
disconnected
,
prefix
,
tmp_prefix
,
user
,
token
from
.
import
platform
,
disconnected
,
prefix
,
tmp_prefix
,
user
,
token
,
temp_cwd
from
..common
import
Selector
from
..scripts.beat
import
main
from
beat.core.test.utils
import
slow
,
cleanup
,
skipif
...
...
@@ -47,15 +47,16 @@ def call(*args, **kwargs):
use_platform
=
kwargs
.
get
(
'platform'
,
platform
)
use_cache
=
kwargs
.
get
(
'cache'
,
'cache'
)
return
main
((
'--platform=%s'
%
use_platform
,
'--user=%s'
%
user
,
'--token=%s'
%
token
,
'--prefix=%s'
%
use_prefix
,
'--cache=%s'
%
use_cache
,
'--test-mode'
,
'algorithm'
,
)
+
args
)
with
temp_cwd
():
return
main
((
'--platform=%s'
%
use_platform
,
'--user=%s'
%
user
,
'--token=%s'
%
token
,
'--prefix=%s'
%
use_prefix
,
'--cache=%s'
%
use_cache
,
'--test-mode'
,
'algorithm'
,
)
+
args
)
@
slow
...
...
beat/cmdline/test/test_cache.py
View file @
4eb87491
...
...
@@ -32,7 +32,7 @@ import os
import
nose.tools
from
.
import
prefix
,
tmp_prefix
from
.
import
prefix
,
tmp_prefix
,
temp_cwd
from
.utils
import
index_experiment_dbs
from
..scripts.beat
import
main
...
...
@@ -45,11 +45,12 @@ def call(*args, **kwargs):
use_prefix
=
kwargs
.
get
(
'prefix'
,
prefix
)
return
main
((
'--prefix=%s'
%
use_prefix
,
'--cache=%s'
%
tmp_prefix
,
'--test-mode'
,
)
+
args
)
with
temp_cwd
():
return
main
((
'--prefix=%s'
%
use_prefix
,
'--cache=%s'
%
tmp_prefix
,
'--test-mode'
,
)
+
args
)
def
setup_module
():
...
...
beat/cmdline/test/test_config.py
View file @
4eb87491
...
...
@@ -34,7 +34,7 @@ import nose.tools
from
nose.tools
import
assert_raises
import
simplejson
from
.
import
tmp_prefix
from
.
import
tmp_prefix
,
temp_cwd
from
..scripts.beat
import
main
from
beat.core.test.utils
import
cleanup
from
..
import
config
...
...
@@ -52,11 +52,12 @@ def call(*args, **kwargs):
def
test_config_list
():
nose
.
tools
.
eq_
(
call
(
'config'
,
'
list
'
),
0
)
nose
.
tools
.
eq_
(
call
(
'config'
,
'
show
'
),
0
)
def
test_config_cache
():
cache_dir
=
'cache'
c
=
config
.
Configuration
({
'--cache'
:
cache_dir
})
nose
.
tools
.
eq_
(
c
.
cache
,
os
.
path
.
join
(
c
.
path
,
cache_dir
))
...
...
@@ -66,48 +67,42 @@ def test_config_cache():
@
nose
.
tools
.
with_setup
(
teardown
=
cleanup
)
def
test_config_save
():
nose
.
tools
.
eq_
(
call
(
'config'
,
'save'
),
0
)
config
=
os
.
path
.
join
(
tmp_prefix
,
'.beat'
,
'config.json'
)
assert
os
.
path
.
exists
(
config
)
with
open
(
config
,
'rt'
)
as
f
:
contents
=
simplejson
.
load
(
f
)
assert
contents
@
nose
.
tools
.
with_setup
(
teardown
=
cleanup
)
def
test_set_token
():
def
test_set_local_token
():
token_value
=
'123456abcdef'
nose
.
tools
.
eq_
(
call
(
'config'
,
'set'
,
'token'
,
token_value
),
0
)
config
=
os
.
path
.
join
(
tmp_prefix
,
'.beat'
,
'config.json'
)
assert
os
.
path
.
exists
(
config
)
with
open
(
config
,
'rt'
)
as
f
:
contents
=
simplejson
.
load
(
f
)
assert
contents
[
'token'
]
==
token_value
with
temp_cwd
()
as
d
:
nose
.
tools
.
eq_
(
call
(
'config'
,
'set'
,
'--local'
,
'token'
,
token_value
),
0
)
config
=
os
.
path
.
join
(
d
,
'.beatrc'
)
assert
os
.
path
.
exists
(
config
)
with
open
(
config
,
'rt'
)
as
f
:
contents
=
simplejson
.
load
(
f
)
assert
contents
[
'token'
]
==
token_value
@
nose
.
tools
.
with_setup
(
teardown
=
cleanup
)
def
test_set_atnt_db
():
def
test_set_
local_
atnt_db
():
db_config
=
'database/atnt'
db_path
=
'./atnt_db'
nose
.
tools
.
eq_
(
call
(
'config'
,
'set'
,
db_config
,
db_path
),
0
)
config
=
os
.
path
.
join
(
tmp_prefix
,
'.beat'
,
'config.json'
)
assert
os
.
path
.
exists
(
config
)
with
open
(
config
,
'rt'
)
as
f
:
contents
=
simplejson
.
load
(
f
)
assert
contents
[
db_config
]
==
db_path
with
temp_cwd
()
as
d
:
nose
.
tools
.
eq_
(
call
(
'config'
,
'set'
,
'--local'
,
db_config
,
db_path
),
0
)
config
=
os
.
path
.
join
(
d
,
'.beatrc'
)