Commit d1be9948 authored by Samuel GAIST's avatar Samuel GAIST

[scripts] Pre-commit cleanup

parent f85433d0
...@@ -56,42 +56,38 @@ options: ...@@ -56,42 +56,38 @@ options:
""" """
import json
import logging import logging
import os import os
import pwd
import sys import sys
import docopt import docopt
import simplejson import simplejson
import pwd
import stat
import json
import zmq
from beat.backend.python.exceptions import UserError
from beat.backend.python.execution import DBExecutor from beat.backend.python.execution import DBExecutor
from beat.backend.python.execution import MessageHandler from beat.backend.python.execution import MessageHandler
from beat.backend.python.exceptions import UserError
# ----------------------------------------------------------
#----------------------------------------------------------
def process_traceback(tb, prefix): def process_traceback(tb, prefix):
import traceback import traceback
databases_prefix = os.path.join(prefix, 'databases') + os.sep databases_prefix = os.path.join(prefix, "databases") + os.sep
for first_line, line in enumerate(tb): for first_line, line in enumerate(tb):
if line[0].startswith(databases_prefix): if line[0].startswith(databases_prefix):
break break
s = ''.join(traceback.format_list(tb[first_line:])) s = "".join(traceback.format_list(tb[first_line:]))
s = s.replace(databases_prefix, 'databases' + os.sep).strip() s = s.replace(databases_prefix, "databases" + os.sep).strip()
return s return s
#---------------------------------------------------------- # ----------------------------------------------------------
def main(arguments=None): def main(arguments=None):
...@@ -100,112 +96,123 @@ def main(arguments=None): ...@@ -100,112 +96,123 @@ def main(arguments=None):
if arguments is None: if arguments is None:
arguments = sys.argv[1:] arguments = sys.argv[1:]
package = __name__.rsplit('.', 2)[0] package = __name__.rsplit(".", 2)[0]
version = package + ' v' + \ version = package + " v" + __import__("pkg_resources").require(package)[0].version
__import__('pkg_resources').require(package)[0].version
prog = os.path.basename(sys.argv[0]) prog = os.path.basename(sys.argv[0])
args = docopt.docopt( args = docopt.docopt(
__doc__ % dict(prog=prog, version=version), __doc__ % dict(prog=prog, version=version), argv=arguments, version=version
argv=arguments,
version=version
) )
# Setup the logging system # Setup the logging system
formatter = logging.Formatter(fmt="[%(asctime)s - databases_provider.py - " \ formatter = logging.Formatter(
"%(name)s] %(levelname)s: %(message)s", fmt="[%(asctime)s - databases_provider.py - "
datefmt="%d/%b/%Y %H:%M:%S") "%(name)s] %(levelname)s: %(message)s",
datefmt="%d/%b/%Y %H:%M:%S",
)
handler = logging.StreamHandler() handler = logging.StreamHandler()
handler.setFormatter(formatter) handler.setFormatter(formatter)
root_logger = logging.getLogger('beat.backend.python') root_logger = logging.getLogger("beat.backend.python")
root_logger.addHandler(handler) root_logger.addHandler(handler)
if args['--debug']: if args["--debug"]:
root_logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG)
else: else:
root_logger.setLevel(logging.INFO) root_logger.setLevel(logging.INFO)
logger = logging.getLogger(__name__)
# Create the message handler # Create the message handler
message_handler = MessageHandler(args['<addr>']) message_handler = MessageHandler(args["<addr>"])
# If necessary, change to another user (with less privileges, but has access # If necessary, change to another user (with less privileges, but has access
# to the databases) # to the databases)
with open(os.path.join(args['<dir>'], 'configuration.json'), 'r') as f: with open(os.path.join(args["<dir>"], "configuration.json"), "r") as f:
cfg = simplejson.load(f) cfg = simplejson.load(f)
if 'datasets_uid' in cfg: if "datasets_uid" in cfg:
# First create the user (if it doesn't exists) # First create the user (if it doesn't exists)
try: try:
user = pwd.getpwuid(cfg['datasets_uid']) _ = pwd.getpwuid(cfg["datasets_uid"])
except: except Exception:
import subprocess import subprocess # nosec
retcode = subprocess.call(['adduser', '--uid', str(cfg['datasets_uid']),
'--no-create-home', '--disabled-password', retcode = subprocess.call( # nosec
'--disabled-login', '--gecos', '""', '-q', [
'beat-nobody']) "adduser",
"--uid",
str(cfg["datasets_uid"]),
"--no-create-home",
"--disabled-password",
"--disabled-login",
"--gecos",
'""',
"-q",
"beat-nobody",
]
)
if retcode != 0: if retcode != 0:
message_handler.send_error('Failed to create an user with the UID %s' % args['uid'], 'sys') message_handler.send_error(
"Failed to create an user with the UID %s" % args["uid"], "sys"
)
return 1 return 1
# Change the current user # Change the current user
try: try:
os.setgid(cfg['datasets_uid']) os.setgid(cfg["datasets_uid"])
os.setuid(cfg['datasets_uid']) os.setuid(cfg["datasets_uid"])
except: except Exception:
import traceback import traceback
message_handler.send_error(traceback.format_exc(), 'sys')
message_handler.send_error(traceback.format_exc(), "sys")
message_handler.destroy() message_handler.destroy()
return 1 return 1
try: try:
# Check the dir # Check the dir
if not os.path.exists(args['<dir>']): if not os.path.exists(args["<dir>"]):
raise IOError("Running directory `%s' not found" % args['<dir>']) raise IOError("Running directory `%s' not found" % args["<dir>"])
# Sets up the execution # Sets up the execution
dataformat_cache = {} dataformat_cache = {}
database_cache = {} database_cache = {}
try: try:
configuration_path = os.path.join(args['<dir>'], 'configuration.json') configuration_path = os.path.join(args["<dir>"], "configuration.json")
if not os.path.exists(configuration_path): if not os.path.exists(configuration_path):
raise RuntimeError("Configuration file '%s' not found" % raise RuntimeError(
configuration_path) "Configuration file '%s' not found" % configuration_path
)
with open(configuration_path) as f: with open(configuration_path) as f:
configuration_data = json.load(f) configuration_data = json.load(f)
configuration_name = args.get('<conf_name>', None) configuration_name = args.get("<conf_name>", None)
if configuration_name is not None: if configuration_name is not None:
configuration_data = configuration_data.get(configuration_name, configuration_data = configuration_data.get(configuration_name, None)
None)
if configuration_data is None: if configuration_data is None:
raise RuntimeError("Configuration section '%s' not found" % raise RuntimeError(
configuration_name) "Configuration section '%s' not found" % configuration_name
)
dbexecutor = DBExecutor(message_handler,
os.path.join(args['<dir>'], 'prefix'), dbexecutor = DBExecutor(
args['<cache>'], message_handler,
configuration_data, os.path.join(args["<dir>"], "prefix"),
dataformat_cache, args["<cache>"],
database_cache) configuration_data,
dataformat_cache,
database_cache,
)
except (MemoryError): except (MemoryError):
raise raise
except Exception as e: except Exception as e:
import traceback import traceback
exc_type, exc_value, exc_traceback = sys.exc_info() exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.extract_tb(exc_traceback) tb = traceback.extract_tb(exc_traceback)
s = process_traceback(tb, os.path.join(args['<dir>'], 'prefix')) s = process_traceback(tb, os.path.join(args["<dir>"], "prefix"))
raise UserError("%s%s: %s" % (s, type(e).__name__, e)) raise UserError("%s%s: %s" % (s, type(e).__name__, e))
# Execute the code # Execute the code
...@@ -216,29 +223,33 @@ def main(arguments=None): ...@@ -216,29 +223,33 @@ def main(arguments=None):
raise raise
except Exception as e: except Exception as e:
import traceback import traceback
exc_type, exc_value, exc_traceback = sys.exc_info() exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.extract_tb(exc_traceback) tb = traceback.extract_tb(exc_traceback)
s = process_traceback(tb, os.path.join(args['<dir>'], 'prefix')) s = process_traceback(tb, os.path.join(args["<dir>"], "prefix"))
raise UserError("%s%s: %s" % (s, type(e).__name__, e)) raise UserError("%s%s: %s" % (s, type(e).__name__, e))
except UserError as e: except UserError as e:
message_handler.send_error(str(e), 'usr') message_handler.send_error(str(e), "usr")
message_handler.destroy() message_handler.destroy()
return 1 return 1
except MemoryError as e: except MemoryError:
# Say something meaningful to the user # Say something meaningful to the user
msg = "The user process for this block ran out of memory. We " \ msg = (
"suggest you optimise your code to reduce memory usage or, " \ "The user process for this block ran out of memory. We "
"if this is not an option, choose an appropriate processing " \ "suggest you optimise your code to reduce memory usage or, "
"queue with enough memory." "if this is not an option, choose an appropriate processing "
message_handler.send_error(msg, 'usr') "queue with enough memory."
)
message_handler.send_error(msg, "usr")
message_handler.destroy() message_handler.destroy()
return 1 return 1
except Exception as e: except Exception:
import traceback import traceback
message_handler.send_error(traceback.format_exc(), 'sys')
message_handler.send_error(traceback.format_exc(), "sys")
message_handler.destroy() message_handler.destroy()
return 1 return 1
...@@ -247,6 +258,5 @@ def main(arguments=None): ...@@ -247,6 +258,5 @@ def main(arguments=None):
return 0 return 0
if __name__ == "__main__":
if __name__ == '__main__':
sys.exit(main()) sys.exit(main())
...@@ -34,46 +34,47 @@ ...@@ -34,46 +34,47 @@
################################################################################### ###################################################################################
import sys import collections
import os import os
import platform import platform
import collections import sys
import simplejson import simplejson
def main(): def main():
# resolve package name # resolve package name
name = 'environment' name = "environment"
if len(sys.argv) > 1: if len(sys.argv) > 1:
name = sys.argv[1] name = sys.argv[1]
# resolve version # resolve version
version = '1' version = "1"
if len(sys.argv) > 2: if len(sys.argv) > 2:
version = sys.argv[2] version = sys.argv[2]
# use a configuration file if one exists # use a configuration file if one exists
databases = None databases = None
capabilities = None capabilities = None
if os.path.exists('/etc/beat/environment.json'): if os.path.exists("/etc/beat/environment.json"):
with open('/etc/beat/environment.json', 'r') as config_file: with open("/etc/beat/environment.json", "r") as config_file:
config = simplejson.load(config_file) config = simplejson.load(config_file)
name = config.get('name', name) name = config.get("name", name)
version = config.get('version', version) version = config.get("version", version)
databases = config.get('databases', None) databases = config.get("databases", None)
capabilities = config.get('capabilities', None) capabilities = config.get("capabilities", None)
# print the result # print the result
retval = collections.OrderedDict() retval = collections.OrderedDict()
retval['name'] = name retval["name"] = name
retval['version'] = version retval["version"] = version
retval['os'] = platform.uname() retval["os"] = platform.uname()
if databases is not None: if databases is not None:
retval['databases'] = databases retval["databases"] = databases
if capabilities is not None: if capabilities is not None:
retval['capabilities'] = capabilities retval["capabilities"] = capabilities
print(simplejson.dumps(retval, indent=2)) print(simplejson.dumps(retval, indent=2))
...@@ -58,18 +58,16 @@ options: ...@@ -58,18 +58,16 @@ options:
""" """
import logging import logging
import os import os
import subprocess # nosec
import sys import sys
import docopt import docopt
import simplejson import simplejson
import subprocess # nosec
import zmq import zmq
from beat.backend.python.execution import AlgorithmExecutor
from beat.backend.python.exceptions import UserError from beat.backend.python.exceptions import UserError
from beat.backend.python.execution import AlgorithmExecutor
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -59,18 +59,17 @@ options: ...@@ -59,18 +59,17 @@ options:
""" """
import logging import logging
import os import os
import pwd
import sys import sys
import docopt import docopt
import pwd
from ..database import Database from ..database import Database
from ..hash import hashDataset from ..hash import hashDataset
from ..hash import toPath from ..hash import toPath
# ----------------------------------------------------------
#----------------------------------------------------------
def main(arguments=None): def main(arguments=None):
...@@ -79,105 +78,114 @@ def main(arguments=None): ...@@ -79,105 +78,114 @@ def main(arguments=None):
if arguments is None: if arguments is None:
arguments = sys.argv[1:] arguments = sys.argv[1:]
package = __name__.rsplit('.', 2)[0] package = __name__.rsplit(".", 2)[0]
version = package + ' v' + \ version = package + " v" + __import__("pkg_resources").require(package)[0].version
__import__('pkg_resources').require(package)[0].version
prog = os.path.basename(sys.argv[0]) prog = os.path.basename(sys.argv[0])
args = docopt.docopt( args = docopt.docopt(
__doc__ % dict(prog=prog, version=version), __doc__ % dict(prog=prog, version=version), argv=arguments, version=version
argv=arguments,
version=version
) )
# Setup the logging system # Setup the logging system
formatter = logging.Formatter(fmt="[%(asctime)s - index.py - " \ formatter = logging.Formatter(
"%(name)s] %(levelname)s: %(message)s", fmt="[%(asctime)s - index.py - " "%(name)s] %(levelname)s: %(message)s",
datefmt="%d/%b/%Y %H:%M:%S") datefmt="%d/%b/%Y %H:%M:%S",
)
handler = logging.StreamHandler() handler = logging.StreamHandler()
handler.setFormatter(formatter) handler.setFormatter(formatter)
root_logger = logging.getLogger('beat.backend.python') root_logger = logging.getLogger("beat.backend.python")
root_logger.addHandler(handler) root_logger.addHandler(handler)
if args['--debug']: if args["--debug"]:
root_logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG)
else: else:
root_logger.setLevel(logging.INFO) root_logger.setLevel(logging.INFO)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if args["--uid"]:
if args['--uid']: uid = int(args["--uid"])
uid = int(args['--uid'])
# First create the user (if it doesn't exists) # First create the user (if it doesn't exists)
try: try:
user = pwd.getpwuid(uid) _ = pwd.getpwuid(uid)
except: except Exception:
import subprocess import subprocess # nosec
retcode = subprocess.call(['adduser', '--uid', str(uid),
'--no-create-home', '--disabled-password', retcode = subprocess.call( # nosec
'--disabled-login', '--gecos', '""', '-q', [
'beat-nobody']) "adduser",
"--uid",
str(uid),
"--no-create-home",
"--disabled-password",
"--disabled-login",
"--gecos",
'""',
"-q",
"beat-nobody",
]
)
if retcode != 0: if retcode != 0:
logger.error('Failed to create an user with the UID %d' % uid) logger.error("Failed to create an user with the UID %d" % uid)
return 1 return 1
# Change the current user # Change the current user
try: try:
os.setgid(uid) os.setgid(uid)
os.setuid(uid) os.setuid(uid)
except: except Exception:
import traceback import traceback
logger.error(traceback.format_exc()) logger.error(traceback.format_exc())
return 1 return 1
# Check the paths # Check the paths
if not os.path.exists(args['<prefix>']): if not os.path.exists(args["<prefix>"]):
logger.error('Invalid prefix path: %s' % args['<prefix>']) logger.error("Invalid prefix path: %s" % args["<prefix>"])
return 1 return 1
if not os.path.exists(args['<cache>']): if not os.path.exists(args["<cache>"]):
logger.error('Invalid cache path: %s' % args['<cache>']) logger.error("Invalid cache path: %s" % args["<cache>"])
return 1 return 1
# Indexing # Indexing
try: try:
database = Database(args['<prefix>'], args['<database>']) database = Database(args["<prefix>"], args["<database>"])
if args['<protocol>'] is None: if args["<protocol>"] is None:
protocols = database.protocol_names protocols = database.protocol_names
else: else:
protocols = [ args['<protocol>'] ] protocols = [args["<protocol>"]]
for protocol in protocols: for protocol in protocols:
if args['<set>'] is None: if args["<set>"] is None:
sets = database.set_names(protocol) sets = database.set_names(protocol)
else: else:
sets = [ args['<set>'] ] sets = [args["<set>"]]
for set_name in sets: for set_name in sets:
filename = toPath(hashDataset(args['<database>'], protocol, set_name), filename = toPath(
suffix='.db') hashDataset(args["<database>"], protocol, set_name), suffix=".db"
)
view = database.view(protocol, set_name, root_folder=args['--db_root_folder']) view = database.view(
view.index(os.path.join(args['<cache>'], filename)) protocol, set_name, root_folder=args["--db_root_folder"]
)
view.index(os.path.join(args["<cache>"], filename))
except Exception as e: except Exception:
import traceback import traceback
logger.error(traceback.format_exc()) logger.error(traceback.format_exc())
return 1 return 1
return 0 return 0