Commit b38536e6 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Merge branch 'pre_commit_cleanup' into 'master'

Pre commit cleanup

See merge request !96
parents e0d0d91d 63c310c9
Pipeline #40959 passed with stages
in 5 minutes and 34 seconds
......@@ -6,12 +6,10 @@ repos:
hooks:
- id: isort
args: [-sl]
- repo: https://github.com/ambv/black
- repo: https://github.com/psf/black
rev: stable
hooks:
- id: black
language_version: python3.6
exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.2.1
hooks:
......@@ -20,6 +18,7 @@ repos:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: debug-statements
exclude: beat/cmdline/test/utils.py
- id: check-added-large-files
- id: check-docstring-first
- id: flake8
......
......@@ -36,4 +36,5 @@
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
......@@ -34,26 +34,24 @@
###################################################################################
import click
import logging
import os
import simplejson as json
import click
import simplejson as json
from beat.backend.python.algorithm import Storage as AlgorithmStorage
from beat.core import algorithm
from beat.core.execution import DockerExecutor
from beat.core.dock import Host
from beat.core import hash
from beat.backend.python.algorithm import Storage as AlgorithmStorage
from beat.core.dock import Host
from beat.core.execution import DockerExecutor
from . import common
from . import commands
from .decorators import raise_on_error
from . import common
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
from .decorators import raise_on_error
logger = logging.getLogger(__name__)
......
......@@ -34,20 +34,20 @@
###################################################################################
import os
import click
import fnmatch
import logging
import os
import click
import simplejson
from beat.core.data import CachedDataSource, load_data_index
from beat.core.data import CachedDataSource
from beat.core.data import load_data_index
from beat.core.utils import NumpyJSONEncoder
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
from .decorators import raise_on_error
logger = logging.getLogger(__name__)
......
......@@ -33,13 +33,13 @@
# #
###################################################################################
import click
import types
from .scripts import main_cli
from .decorators import raise_on_error
import click
from . import common
from .decorators import raise_on_error
from .scripts import main_cli
def copy_func(f, name=None):
......
......@@ -36,29 +36,29 @@
"""Utility functions that are useful to all sub-commands"""
import os
import glob
import fnmatch
import difflib
import collections
import difflib
import fnmatch
import glob
import logging
import os
from enum import Enum
from enum import unique
import simplejson
import six
import termcolor
import simplejson
from enum import Enum, unique
from beat.core import dataformat
from beat.core import algorithm
from beat.core import database
from beat.core import dataformat
from beat.core import experiment
from beat.core import library
from beat.core import plotter
from beat.core import plotterparameter
from beat.core import protocoltemplate
from beat.core import algorithm
from beat.core import toolchain
from beat.core import experiment
logger = logging.getLogger(__name__)
......
......@@ -37,91 +37,85 @@
"""Configuration manipulation and display"""
import os
import sys
import copy
import logging
import getpass
import logging
import os
import sys
import click
import simplejson
from .decorators import verbosity_option
from .click_helper import AliasedGroup
from .decorators import verbosity_option
logger = logging.getLogger(__name__)
# Default values for the command-line utility
DEFAULTS = {
'platform': 'https://www.beat-eu.org/platform/',
'user': getpass.getuser(),
'token': None,
'prefix': os.path.realpath(os.path.join(os.curdir, 'prefix')),
'cache': 'cache',
'editor': None,
}
"""Default values for the command-line utility"""
"platform": "https://www.beat-eu.org/platform/",
"user": getpass.getuser(),
"token": None,
"prefix": os.path.realpath(os.path.join(os.curdir, "prefix")),
"cache": "cache",
"editor": None,
}
# Documentation for the configuration parameters
DOC = {
'platform': 'Web address of the BEAT platform',
'user': 'User name for operations that create, delete or edit objects',
'token': 'Secret key of the user on the BEAT platform',
'prefix': 'Directory containing BEAT objects',
'cache': 'Directory to use for data caching (relative to prefix)',
'editor': 'Editor to be used to edit local files',
}
"""Documentation for configuration parameters"""
"platform": "Web address of the BEAT platform",
"user": "User name for operations that create, delete or edit objects",
"token": "Secret key of the user on the BEAT platform",
"prefix": "Directory containing BEAT objects",
"cache": "Directory to use for data caching (relative to prefix)",
"editor": "Editor to be used to edit local files",
}
class Configuration(object):
'''Keeps track of configuration elements'''
"""Keeps track of configuration elements"""
def __init__(self, args):
self.files = [
os.path.expanduser('~/.beatrc'),
os.path.realpath('./.beatrc'),
os.path.expanduser("~/.beatrc"),
os.path.realpath("./.beatrc"),
]
self.__data = copy.deepcopy(DEFAULTS)
for k in self.files:
if os.path.exists(k):
with open(k, 'rt') as f: tmp = simplejson.load(f)
with open(k, "rt") as f:
tmp = simplejson.load(f)
self.__data.update(tmp)
logger.info("Loaded configuration file `%s'", k)
for key in DEFAULTS:
self.__data[key] = args.get('--%s' % key) or self.__data[key]
self.__data[key] = args.get("--%s" % key) or self.__data[key]
@property
def path(self):
'''The directory for the prefix'''
return self.__data['prefix']
"""The directory for the prefix"""
return self.__data["prefix"]
@property
def cache(self):
'''The directory for the cache'''
if os.path.isabs(self.__data['cache']):
return self.__data['cache']
return os.path.join(self.__data['prefix'], self.__data['cache'])
"""The directory for the cache"""
if os.path.isabs(self.__data["cache"]):
return self.__data["cache"]
return os.path.join(self.__data["prefix"], self.__data["cache"])
@property
def database_paths(self):
'''A dict of paths for databases'''
return dict(
(k, self.__data[k]) for k in self.__data if self.is_database_key(k)
)
"""A dict of paths for databases"""
return dict((k, self.__data[k]) for k in self.__data if self.is_database_key(k))
def set(self, key, value, local=False):
'''Sets or resets a field in the configuration'''
"""Sets or resets a field in the configuration"""
if not self._is_valid_key(key):
logger.error("Don't know about parameter `%s'", key)
......@@ -134,9 +128,8 @@ class Configuration(object):
self.save(local)
def save(self, local=False):
'''Saves contents to configuration file
"""Saves contents to configuration file
Parameters:
......@@ -144,39 +137,34 @@ class Configuration(object):
if set to ``True``, then save
configuration values to local configuration file (typically
``.beatrc``)
'''
"""
path = self.files[0]
if local:
path = self.files[1]
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
with os.fdopen(os.open(path, flags, 0o600), 'wt') as f:
f.write(simplejson.dumps(
self.__data, sort_keys=True, indent=4,
separators=(',', ': ')
))
with os.fdopen(os.open(path, flags, 0o600), "wt") as f:
f.write(
simplejson.dumps(
self.__data, sort_keys=True, indent=4, separators=(",", ": ")
)
)
def _is_valid_key(self, key):
return key in DEFAULTS or self.is_database_key(key)
def is_database_key(self, key):
return key.startswith('database/')
return key.startswith("database/")
def __str__(self):
return simplejson.dumps(
self.__data, sort_keys=True,
indent=4, separators=(',', ': ')
self.__data, sort_keys=True, indent=4, separators=(",", ": ")
)
def as_dict(self):
return copy.copy(self.__data)
def __getattr__(self, key):
return self.__data[key]
......@@ -188,6 +176,7 @@ def config(ctx):
"""The manager for beat cmdline configuration."""
pass
@config.command()
@click.pass_context
def show(ctx):
......@@ -195,11 +184,11 @@ def show(ctx):
Lists the configuration after resolving defaults and saved variables
"""
click.echo(ctx.meta['config'])
click.echo(ctx.meta["config"])
@config.command()
@click.argument('key')
@click.argument("key")
@click.pass_context
def get(ctx, key):
"""Prints out the contents of a single field.
......@@ -221,19 +210,22 @@ def get(ctx, key):
-----
* If the key is not found.
"""
value = getattr(ctx.meta['config'], key)
value = getattr(ctx.meta["config"], key)
if value is None:
# Exit the command line with ClickException in case of errors.
raise click.ClickException(
"The requested key `{}' does not exist".format(key))
raise click.ClickException("The requested key `{}' does not exist".format(key))
click.echo(value)
@config.command()
@click.argument('args', nargs=-1)
@click.option('--local/--not-local', default=False, help='Save values on the '
'local configuration file (.beatrc) instead of using the global '
'file (~/.beatrc)')
@click.argument("args", nargs=-1)
@click.option(
"--local/--not-local",
default=False,
help="Save values on the "
"local configuration file (.beatrc) instead of using the global "
"file (~/.beatrc)",
)
@click.pass_context
def set(ctx, args, local):
"""Sets the value for a key.
......@@ -260,10 +252,10 @@ def set(ctx, args, local):
"""
if len(args) % 2 != 0:
raise click.BadParameter('You must provide pair(s) of key/value')
raise click.BadParameter("You must provide pair(s) of key/value")
try:
for idx in range(0, len(args), 2):
ctx.meta['config'].set(args[idx], args[idx + 1], local)
ctx.meta["config"].set(args[idx], args[idx + 1], local)
except Exception:
raise click.ClickException("Failed to change the configuration.")
......@@ -34,31 +34,30 @@
###################################################################################
import os
import click
import glob
import random
import zmq
import logging
import os
import random
import click
import simplejson
import zmq
from beat.core.hash import toPath
from beat.core.hash import hashDataset
from beat.core.utils import NumpyJSONEncoder
from beat.core.database import Database
from beat.core.data import RemoteDataSource
from beat.core import dock
from beat.core import inputs
from beat.core import utils
from beat.core.data import RemoteDataSource
from beat.core.database import Database
from beat.core.hash import hashDataset
from beat.core.hash import toPath
from beat.core.utils import NumpyJSONEncoder
from . import common
from . import commands
from .decorators import raise_on_error
from . import common
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
from .decorators import raise_on_error
logger = logging.getLogger(__name__)
......
......@@ -35,17 +35,17 @@
import logging
import click
from beat.core import dataformat
from . import common
from . import commands
from .decorators import raise_on_error
from . import common
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
from .decorators import raise_on_error
logger = logging.getLogger(__name__)
......
......@@ -34,9 +34,11 @@
import logging
import click
from functools import wraps
import click
from .log import set_verbosity_level
# This needs to be beat so that logger is configured for all beat packages.
......
......@@ -34,17 +34,16 @@
###################################################################################
import click
import logging
import click
import simplejson as json
from beat.core.dock import Host
from . import common
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
......
......@@ -74,18 +74,17 @@ Options:
"""
import logging
import click
from beat.core import library
from . import common
from . import commands
from .decorators import raise_on_error
from . import common
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
from .decorators import raise_on_error
logger = logging.getLogger(__name__)
......
......@@ -5,6 +5,7 @@
import logging
def set_verbosity_level(logger, level):
"""Sets the log level for the given logger.
......@@ -30,7 +31,7 @@ def set_verbosity_level(logger, level):
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
3: logging.DEBUG
3: logging.DEBUG,
}[level]
# set this log level to the logger with the specified name
......
......@@ -36,13 +36,12 @@
import click
from . import common
from . import commands
from .decorators import raise_on_error
from . import common
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
from .decorators import raise_on_error
def pull_impl(webapi, prefix, names, force, indentation, format_cache):
......
......@@ -34,31 +34,30 @@
###################################################################################
import collections
import io
import logging
import os
import sys
import click
import logging
import simplejson
import collections
from PIL import Image
from beat.core import plotter
from beat.core import dataformat
from beat.core import plotter
from beat.core.plotterparameter import Storage as PPStorage
from . import common
from . import commands
from .dataformats import pull_impl as dataformats_pull
from .plotterparameters import pull_impl as plotterparameters_pull
from .libraries import pull_impl as libraries_pull
from .decorators import raise_on_error
from . import common
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
from .dataformats import pull_impl as dataformats_pull
from .decorators import raise_on_error
from .libraries import pull_impl as libraries_pull
from .plotterparameters import pull_impl as plotterparameters_pull
logger = logging.getLogger(__name__)
......
......@@ -34,16 +34,15 @@
###################################################################################
import click
import logging
from . import commands
import click
from . import commands
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
logger = logging.getLogger(__name__)
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
......@@ -2,12 +2,15 @@
"""
import logging
import pkg_resources
import click
import pkg_resources
from click_plugins import with_plugins
from ..click_helper import AliasedGroup
from ..decorators import verbosity_option
from ..config import Configuration
from ..decorators import verbosity_option
# defines our own logging level for extra information to be printed
logging.EXTRA = 15
......
......@@ -51,10 +51,10 @@ from .decorators import raise_on_error
@raise_on_error
def status(ctx):
"""Shows the working folder status for different objects"""
path = ctx.meta['config'].path
with common.make_webapi(ctx.meta['config']) as webapi:
retval = common.status(webapi, path, 'dataformat')[0]
retval += common.status(webapi, path, 'algorithm')[0]
retval += common.status(webapi, path, 'toolchain')[0]
retval += common.status(webapi, path, 'experiment')[0]
path = ctx.meta["config"].path
with common.make_webapi(ctx.meta["config"]) as webapi:
retval = common.status(webapi, path, "dataformat")[0]
retval += common.status(webapi, path, "algorithm")[0]
retval += common.status(webapi, path, "toolchain")[0]
retval += common.status(webapi, path, "experiment")[0]
return retval
......@@ -36,18 +36,18 @@
# Basic setup for command test
import contextlib
import os
import sys
import tempfile
import shutil
import subprocess # nosec
import pkg_resources
import contextlib
import sys
import tempfile
import urllib
from beat.core.test import tmp_prefix # noqa forward import
import pkg_resources
from beat.core.test import teardown_package as bc_teardown_package
from beat.core.test import tmp_prefix # noqa forward import
platform = os.environ.get("BEAT_CMDLINE_TEST_PLATFORM", "")
......
......@@ -39,23 +39,28 @@ Base class for asset testing
"""
import os
import nose.tools
import click
import shutil
from collections import namedtuple
from functools import wraps
import click
import nose.tools
from click.testing import CliRunner
from beat.cmdline.scripts import main_cli
from beat.core.test.utils import cleanup
from beat.core.test.utils import skipif
from beat.core.test.utils import slow
from beat.cmdline.scripts import main_cli
from .. import common
from . import platform, disconnected, prefix, tmp_prefix, user, token
from . import disconnected
from . import platform
from . import prefix
from . import tmp_prefix
from . import token
from . import user