Commit d1be9948 authored by Samuel GAIST's avatar Samuel GAIST

[scripts] Pre-commit cleanup

parent f85433d0
......@@ -56,42 +56,38 @@ options:
"""
import json
import logging
import os
import pwd
import sys
import docopt
import simplejson
import pwd
import stat
import json
import zmq
from beat.backend.python.exceptions import UserError
from beat.backend.python.execution import DBExecutor
from beat.backend.python.execution import MessageHandler
from beat.backend.python.exceptions import UserError
#----------------------------------------------------------
# ----------------------------------------------------------
def process_traceback(tb, prefix):
import traceback
databases_prefix = os.path.join(prefix, 'databases') + os.sep
databases_prefix = os.path.join(prefix, "databases") + os.sep
for first_line, line in enumerate(tb):
if line[0].startswith(databases_prefix):
break
s = ''.join(traceback.format_list(tb[first_line:]))
s = s.replace(databases_prefix, 'databases' + os.sep).strip()
s = "".join(traceback.format_list(tb[first_line:]))
s = s.replace(databases_prefix, "databases" + os.sep).strip()
return s
#----------------------------------------------------------
# ----------------------------------------------------------
def main(arguments=None):
......@@ -100,112 +96,123 @@ def main(arguments=None):
if arguments is None:
arguments = sys.argv[1:]
package = __name__.rsplit('.', 2)[0]
version = package + ' v' + \
__import__('pkg_resources').require(package)[0].version
package = __name__.rsplit(".", 2)[0]
version = package + " v" + __import__("pkg_resources").require(package)[0].version
prog = os.path.basename(sys.argv[0])
args = docopt.docopt(
__doc__ % dict(prog=prog, version=version),
argv=arguments,
version=version
__doc__ % dict(prog=prog, version=version), argv=arguments, version=version
)
# Setup the logging system
formatter = logging.Formatter(fmt="[%(asctime)s - databases_provider.py - " \
"%(name)s] %(levelname)s: %(message)s",
datefmt="%d/%b/%Y %H:%M:%S")
formatter = logging.Formatter(
fmt="[%(asctime)s - databases_provider.py - "
"%(name)s] %(levelname)s: %(message)s",
datefmt="%d/%b/%Y %H:%M:%S",
)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger = logging.getLogger('beat.backend.python')
root_logger = logging.getLogger("beat.backend.python")
root_logger.addHandler(handler)
if args['--debug']:
if args["--debug"]:
root_logger.setLevel(logging.DEBUG)
else:
root_logger.setLevel(logging.INFO)
logger = logging.getLogger(__name__)
# Create the message handler
message_handler = MessageHandler(args['<addr>'])
message_handler = MessageHandler(args["<addr>"])
# If necessary, change to another user (with less privileges, but has access
# to the databases)
with open(os.path.join(args['<dir>'], 'configuration.json'), 'r') as f:
with open(os.path.join(args["<dir>"], "configuration.json"), "r") as f:
cfg = simplejson.load(f)
if 'datasets_uid' in cfg:
if "datasets_uid" in cfg:
# First create the user (if it doesn't exists)
try:
user = pwd.getpwuid(cfg['datasets_uid'])
except:
import subprocess
retcode = subprocess.call(['adduser', '--uid', str(cfg['datasets_uid']),
'--no-create-home', '--disabled-password',
'--disabled-login', '--gecos', '""', '-q',
'beat-nobody'])
_ = pwd.getpwuid(cfg["datasets_uid"])
except Exception:
import subprocess # nosec
retcode = subprocess.call( # nosec
[
"adduser",
"--uid",
str(cfg["datasets_uid"]),
"--no-create-home",
"--disabled-password",
"--disabled-login",
"--gecos",
'""',
"-q",
"beat-nobody",
]
)
if retcode != 0:
message_handler.send_error('Failed to create an user with the UID %s' % args['uid'], 'sys')
message_handler.send_error(
"Failed to create an user with the UID %s" % args["uid"], "sys"
)
return 1
# Change the current user
try:
os.setgid(cfg['datasets_uid'])
os.setuid(cfg['datasets_uid'])
except:
os.setgid(cfg["datasets_uid"])
os.setuid(cfg["datasets_uid"])
except Exception:
import traceback
message_handler.send_error(traceback.format_exc(), 'sys')
message_handler.send_error(traceback.format_exc(), "sys")
message_handler.destroy()
return 1
try:
# Check the dir
if not os.path.exists(args['<dir>']):
raise IOError("Running directory `%s' not found" % args['<dir>'])
if not os.path.exists(args["<dir>"]):
raise IOError("Running directory `%s' not found" % args["<dir>"])
# Sets up the execution
dataformat_cache = {}
database_cache = {}
try:
configuration_path = os.path.join(args['<dir>'], 'configuration.json')
configuration_path = os.path.join(args["<dir>"], "configuration.json")
if not os.path.exists(configuration_path):
raise RuntimeError("Configuration file '%s' not found" %
configuration_path)
raise RuntimeError(
"Configuration file '%s' not found" % configuration_path
)
with open(configuration_path) as f:
configuration_data = json.load(f)
configuration_name = args.get('<conf_name>', None)
configuration_name = args.get("<conf_name>", None)
if configuration_name is not None:
configuration_data = configuration_data.get(configuration_name,
None)
configuration_data = configuration_data.get(configuration_name, None)
if configuration_data is None:
raise RuntimeError("Configuration section '%s' not found" %
configuration_name)
dbexecutor = DBExecutor(message_handler,
os.path.join(args['<dir>'], 'prefix'),
args['<cache>'],
configuration_data,
dataformat_cache,
database_cache)
raise RuntimeError(
"Configuration section '%s' not found" % configuration_name
)
dbexecutor = DBExecutor(
message_handler,
os.path.join(args["<dir>"], "prefix"),
args["<cache>"],
configuration_data,
dataformat_cache,
database_cache,
)
except (MemoryError):
raise
except Exception as e:
import traceback
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.extract_tb(exc_traceback)
s = process_traceback(tb, os.path.join(args['<dir>'], 'prefix'))
s = process_traceback(tb, os.path.join(args["<dir>"], "prefix"))
raise UserError("%s%s: %s" % (s, type(e).__name__, e))
# Execute the code
......@@ -216,29 +223,33 @@ def main(arguments=None):
raise
except Exception as e:
import traceback
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.extract_tb(exc_traceback)
s = process_traceback(tb, os.path.join(args['<dir>'], 'prefix'))
s = process_traceback(tb, os.path.join(args["<dir>"], "prefix"))
raise UserError("%s%s: %s" % (s, type(e).__name__, e))
except UserError as e:
message_handler.send_error(str(e), 'usr')
message_handler.send_error(str(e), "usr")
message_handler.destroy()
return 1
except MemoryError as e:
except MemoryError:
# Say something meaningful to the user
msg = "The user process for this block ran out of memory. We " \
"suggest you optimise your code to reduce memory usage or, " \
"if this is not an option, choose an appropriate processing " \
"queue with enough memory."
message_handler.send_error(msg, 'usr')
msg = (
"The user process for this block ran out of memory. We "
"suggest you optimise your code to reduce memory usage or, "
"if this is not an option, choose an appropriate processing "
"queue with enough memory."
)
message_handler.send_error(msg, "usr")
message_handler.destroy()
return 1
except Exception as e:
except Exception:
import traceback
message_handler.send_error(traceback.format_exc(), 'sys')
message_handler.send_error(traceback.format_exc(), "sys")
message_handler.destroy()
return 1
......@@ -247,6 +258,5 @@ def main(arguments=None):
return 0
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())
......@@ -34,46 +34,47 @@
###################################################################################
import sys
import collections
import os
import platform
import collections
import sys
import simplejson
def main():
# resolve package name
name = 'environment'
name = "environment"
if len(sys.argv) > 1:
name = sys.argv[1]
# resolve version
version = '1'
version = "1"
if len(sys.argv) > 2:
version = sys.argv[2]
# use a configuration file if one exists
databases = None
capabilities = None
if os.path.exists('/etc/beat/environment.json'):
with open('/etc/beat/environment.json', 'r') as config_file:
if os.path.exists("/etc/beat/environment.json"):
with open("/etc/beat/environment.json", "r") as config_file:
config = simplejson.load(config_file)
name = config.get('name', name)
version = config.get('version', version)
databases = config.get('databases', None)
capabilities = config.get('capabilities', None)
name = config.get("name", name)
version = config.get("version", version)
databases = config.get("databases", None)
capabilities = config.get("capabilities", None)
# print the result
retval = collections.OrderedDict()
retval['name'] = name
retval['version'] = version
retval['os'] = platform.uname()
retval["name"] = name
retval["version"] = version
retval["os"] = platform.uname()
if databases is not None:
retval['databases'] = databases
retval["databases"] = databases
if capabilities is not None:
retval['capabilities'] = capabilities
retval["capabilities"] = capabilities
print(simplejson.dumps(retval, indent=2))
......@@ -58,18 +58,16 @@ options:
"""
import logging
import os
import subprocess # nosec
import sys
import docopt
import simplejson
import subprocess # nosec
import zmq
from beat.backend.python.execution import AlgorithmExecutor
from beat.backend.python.exceptions import UserError
from beat.backend.python.execution import AlgorithmExecutor
# ----------------------------------------------------------
......
......@@ -59,18 +59,17 @@ options:
"""
import logging
import os
import pwd
import sys
import docopt
import pwd
from ..database import Database
from ..hash import hashDataset
from ..hash import toPath
#----------------------------------------------------------
# ----------------------------------------------------------
def main(arguments=None):
......@@ -79,105 +78,114 @@ def main(arguments=None):
if arguments is None:
arguments = sys.argv[1:]
package = __name__.rsplit('.', 2)[0]
version = package + ' v' + \
__import__('pkg_resources').require(package)[0].version
package = __name__.rsplit(".", 2)[0]
version = package + " v" + __import__("pkg_resources").require(package)[0].version
prog = os.path.basename(sys.argv[0])
args = docopt.docopt(
__doc__ % dict(prog=prog, version=version),
argv=arguments,
version=version
__doc__ % dict(prog=prog, version=version), argv=arguments, version=version
)
# Setup the logging system
formatter = logging.Formatter(fmt="[%(asctime)s - index.py - " \
"%(name)s] %(levelname)s: %(message)s",
datefmt="%d/%b/%Y %H:%M:%S")
formatter = logging.Formatter(
fmt="[%(asctime)s - index.py - " "%(name)s] %(levelname)s: %(message)s",
datefmt="%d/%b/%Y %H:%M:%S",
)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger = logging.getLogger('beat.backend.python')
root_logger = logging.getLogger("beat.backend.python")
root_logger.addHandler(handler)
if args['--debug']:
if args["--debug"]:
root_logger.setLevel(logging.DEBUG)
else:
root_logger.setLevel(logging.INFO)
logger = logging.getLogger(__name__)
if args['--uid']:
uid = int(args['--uid'])
if args["--uid"]:
uid = int(args["--uid"])
# First create the user (if it doesn't exists)
try:
user = pwd.getpwuid(uid)
except:
import subprocess
retcode = subprocess.call(['adduser', '--uid', str(uid),
'--no-create-home', '--disabled-password',
'--disabled-login', '--gecos', '""', '-q',
'beat-nobody'])
_ = pwd.getpwuid(uid)
except Exception:
import subprocess # nosec
retcode = subprocess.call( # nosec
[
"adduser",
"--uid",
str(uid),
"--no-create-home",
"--disabled-password",
"--disabled-login",
"--gecos",
'""',
"-q",
"beat-nobody",
]
)
if retcode != 0:
logger.error('Failed to create an user with the UID %d' % uid)
logger.error("Failed to create an user with the UID %d" % uid)
return 1
# Change the current user
try:
os.setgid(uid)
os.setuid(uid)
except:
except Exception:
import traceback
logger.error(traceback.format_exc())
return 1
# Check the paths
if not os.path.exists(args['<prefix>']):
logger.error('Invalid prefix path: %s' % args['<prefix>'])
if not os.path.exists(args["<prefix>"]):
logger.error("Invalid prefix path: %s" % args["<prefix>"])
return 1
if not os.path.exists(args['<cache>']):
logger.error('Invalid cache path: %s' % args['<cache>'])
if not os.path.exists(args["<cache>"]):
logger.error("Invalid cache path: %s" % args["<cache>"])
return 1
# Indexing
try:
database = Database(args['<prefix>'], args['<database>'])
database = Database(args["<prefix>"], args["<database>"])
if args['<protocol>'] is None:
if args["<protocol>"] is None:
protocols = database.protocol_names
else:
protocols = [ args['<protocol>'] ]
protocols = [args["<protocol>"]]
for protocol in protocols:
if args['<set>'] is None:
if args["<set>"] is None:
sets = database.set_names(protocol)
else:
sets = [ args['<set>'] ]
sets = [args["<set>"]]
for set_name in sets:
filename = toPath(hashDataset(args['<database>'], protocol, set_name),
suffix='.db')
filename = toPath(
hashDataset(args["<database>"], protocol, set_name), suffix=".db"
)
view = database.view(protocol, set_name, root_folder=args['--db_root_folder'])
view.index(os.path.join(args['<cache>'], filename))
view = database.view(
protocol, set_name, root_folder=args["--db_root_folder"]
)
view.index(os.path.join(args["<cache>"], filename))
except Exception as e:
except Exception:
import traceback
logger.error(traceback.format_exc())
return 1
return 0
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())
......@@ -57,19 +57,17 @@ options:
"""
import logging
import os
import subprocess # nosec
import sys
import docopt
import simplejson as json
import subprocess # nosec
import zmq
from beat.backend.python.exceptions import UserError
from beat.backend.python.execution import LoopExecutor
from beat.backend.python.execution import LoopMessageHandler
from beat.backend.python.exceptions import UserError
# ----------------------------------------------------------
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment