[algorithms] pre-commit cleanup

beat/cmdline/algorithms.py

@@ -55,7 +55,7 @@ logger = logging.getLogger(__name__)
 
 
 def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache):
-  """Copies algorithms (and required libraries/dataformats) from the server.
+    """Copies algorithms (and required libraries/dataformats) from the server.
 
   Parameters:
 
@@ -93,37 +93,48 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache
 
   """
 
-  from .dataformats import pull_impl as dataformats_pull
-  from .libraries import pull_impl as libraries_pull
+    from .dataformats import pull_impl as dataformats_pull
+    from .libraries import pull_impl as libraries_pull
 
-  status, names = common.pull(webapi, prefix, 'algorithm', names,
-                              ['declaration', 'code', 'description'], force, indentation)
+    status, names = common.pull(
+        webapi,
+        prefix,
+        "algorithm",
+        names,
+        ["declaration", "code", "description"],
+        force,
+        indentation,
+    )
 
-  if status != 0: return status
+    if status != 0:
+        return status
 
-  # see what dataformats one needs to pull
-  indent = indentation * ' '
+    # see what dataformats one needs to pull
+    if indentation == 0:
+        indentation = 4
 
-  dataformats = []
-  libraries = []
-  for name in names:
-    obj = algorithm.Algorithm(prefix, name)
-    dataformats.extend(obj.dataformats.keys())
-    libraries.extend(obj.libraries.keys())
+    dataformats = []
+    libraries = []
+    for name in names:
+        obj = algorithm.Algorithm(prefix, name)
+        dataformats.extend(obj.dataformats.keys())
+        libraries.extend(obj.libraries.keys())
 
-  # downloads any formats to which we depend on
-  df_status = dataformats_pull(webapi, prefix, dataformats, force,
-                               indentation + 2, format_cache)
+    # downloads any formats to which we depend on
+    df_status = dataformats_pull(
+        webapi, prefix, dataformats, force, indentation, format_cache
+    )
 
-  lib_status = libraries_pull(webapi, prefix, libraries, force,
-                              indentation + 2, lib_cache)
-
-  return status + df_status + lib_status
+    lib_status = libraries_pull(
+        webapi, prefix, libraries, force, indentation, lib_cache
+    )
 
+    return status + df_status + lib_status
 
 
 def print_examples():
-    print("""
+    print(
+        """
 To feed data from a database to an algorithm:
 =============================================
 
@@ -201,99 +212,115 @@ To execute an analyzer:
         "version": "<environment_version>"
     }
 }
-""")
-
+"""
+    )
 
 
 def execute_impl(prefix, cache, instructions_file):
-  try:
-    # Load the JSON configuration
-    if not os.path.exists(instructions_file):
-      raise IOError("JSON instructions file `%s' not found" % instructions_file)
-
-    with open(instructions_file, 'r') as f:
-      configuration = json.load(f)
-
-    # Add missing configuration fields
-    configuration['queue'] = 'unused'
-    configuration['nb_slots'] = 1
-
-    if 'parameters' not in configuration:
-      configuration['parameters'] = {}
-
-    for name, cfg in configuration['inputs'].items():
-      cfg['endpoint'] = name
-      suffix = ''
-      if 'database' in cfg:   # Connected to a database output
-        cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set'])
-        suffix = '.db'
-
-      cfg['path'] = hash.toPath(cfg['hash'], suffix=suffix)
-
-    algo = AlgorithmStorage(prefix, configuration['algorithm'])
-
-    if 'outputs' in  configuration:   # Standard algorithm
-      for name, cfg in configuration['outputs'].items():
-          cfg['endpoint'] = name
-          cfg['hash'] = hash.hashBlockOutput(
-              'block',
-              configuration['algorithm'],
-              algo.hash(),
-              configuration['parameters'],
-              configuration['environment'],
-              dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
-              name,
-          )
-          cfg['path'] = hash.toPath(cfg['hash'], '')
-
-    else:   # Analyzer
-      configuration['result'] = {}
-      configuration['result']['hash'] = hash.hashAnalyzer(
-          'block',
-              configuration['algorithm'],
-              algo.hash(),
-              configuration['parameters'],
-              configuration['environment'],
-              dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
-      )
-      configuration['result']['path'] = hash.toPath(configuration['result']['hash'], '')
-
-    # Sets up the execution
-    dataformat_cache = {}
-    database_cache = {}
-    algorithm_cache = {}
-
-    host = Host(raise_on_errors=False)
-
-    executor = DockerExecutor(host, prefix, configuration, cache, dataformat_cache,
-                              database_cache, algorithm_cache)
-
-    if not executor.valid:
-      logger.error('Invalid configuration:\n  * %s' % '\n  * '.join(executor.errors))
-      return 1
-
-    # Execute the algorithm
-    with executor:
-      result = executor.process()
-      if result['status'] != 0:
-          print('STDERR:')
-          print(result['stderr'])
-
-    # Display the results
-    if 'outputs' in configuration:   # Standard algorithm
-      print('Outputs of the algorithms available at:')
-      for name, cfg in configuration['outputs'].items():
-        print('  - %s: %s' % (name, cfg['path']))
-    else:
-      print('Results of the analyzer available at: %s' % configuration['result']['path'])
-
-  except Exception as e:
-    import traceback
-    logger.error(traceback.format_exc())
-    return 1
-
-  return 0
+    try:
+        # Load the JSON configuration
+        if not os.path.exists(instructions_file):
+            raise IOError("JSON instructions file `%s' not found" % instructions_file)
+
+        with open(instructions_file, "r") as f:
+            configuration = json.load(f)
+
+        # Add missing configuration fields
+        configuration["queue"] = "unused"
+        configuration["nb_slots"] = 1
+
+        if "parameters" not in configuration:
+            configuration["parameters"] = {}
+
+        for name, cfg in configuration["inputs"].items():
+            cfg["endpoint"] = name
+            suffix = ""
+            if "database" in cfg:  # Connected to a database output
+                cfg["hash"] = hash.hashDataset(
+                    cfg["database"], cfg["protocol"], cfg["set"]
+                )
+                suffix = ".db"
+
+            cfg["path"] = hash.toPath(cfg["hash"], suffix=suffix)
+
+        algo = AlgorithmStorage(prefix, configuration["algorithm"])
+
+        if "outputs" in configuration:  # Standard algorithm
+            for name, cfg in configuration["outputs"].items():
+                cfg["endpoint"] = name
+                cfg["hash"] = hash.hashBlockOutput(
+                    "block",
+                    configuration["algorithm"],
+                    algo.hash(),
+                    configuration["parameters"],
+                    configuration["environment"],
+                    dict([(k, v["hash"]) for k, v in configuration["inputs"].items()]),
+                    name,
+                )
+                cfg["path"] = hash.toPath(cfg["hash"], "")
+
+        else:  # Analyzer
+            configuration["result"] = {}
+            configuration["result"]["hash"] = hash.hashAnalyzer(
+                "block",
+                configuration["algorithm"],
+                algo.hash(),
+                configuration["parameters"],
+                configuration["environment"],
+                dict([(k, v["hash"]) for k, v in configuration["inputs"].items()]),
+            )
+            configuration["result"]["path"] = hash.toPath(
+                configuration["result"]["hash"], ""
+            )
+
+        # Sets up the execution
+        dataformat_cache = {}
+        database_cache = {}
+        algorithm_cache = {}
+
+        host = Host(raise_on_errors=False)
+
+        executor = DockerExecutor(
+            host,
+            prefix,
+            configuration,
+            cache,
+            dataformat_cache,
+            database_cache,
+            algorithm_cache,
+        )
+
+        if not executor.valid:
+            logger.error(
+                "Invalid configuration:\n  * %s" % "\n  * ".join(executor.errors)
+            )
+            return 1
+
+        # Execute the algorithm
+        with executor:
+            result = executor.process()
+            if result["status"] != 0:
+                print("STDERR:")
+                print(result["stderr"])
+
+        # Display the results
+        if "outputs" in configuration:  # Standard algorithm
+            print("Outputs of the algorithms available at:")
+            for name, cfg in configuration["outputs"].items():
+                print("  - %s: %s" % (name, cfg["path"]))
+        else:
+            print(
+                "Results of the analyzer available at: %s"
+                % configuration["result"]["path"]
+            )
+
+    except Exception:
+        import traceback
+
+        logger.error(traceback.format_exc())
+        return 1
 
+    return 0
 
 
 @click.group(cls=AliasedGroup)
@@ -302,213 +329,225 @@ def algorithms(ctx):
     """Configuration and manipulation of algorithms"""
     pass
 
+
 @algorithms.command()
-@click.option('--remote', help='Only acts on the remote copy of the algorithm',
-              is_flag=True)
+@click.option(
+    "--remote", help="Only acts on the remote copy of the algorithm", is_flag=True
+)
 @click.pass_context
 @raise_on_error
 def list(ctx, remote):
-  '''Lists all the algorithms available on the platform
+    """Lists all the algorithms available on the platform
 
   Example:
     $ beat algorithms list --remote
-  '''
-  if remote:
-    with common.make_webapi(ctx.meta['config']) as webapi:
-      return common.display_remote_list(webapi, 'algorithm')
-  else:
-    return common.display_local_list(ctx.meta['config'].path, 'algorithm')
+  """
+    if remote:
+        with common.make_webapi(ctx.meta["config"]) as webapi:
+            return common.display_remote_list(webapi, "algorithm")
+    else:
+        return common.display_local_list(ctx.meta["config"].path, "algorithm")
+
 
 @algorithms.command()
-@click.argument('names', nargs=-1)
+@click.argument("names", nargs=-1)
 @click.pass_context
 @raise_on_error
 def path(ctx, names):
-  '''Displays local path of algorithm files
+    """Displays local path of algorithm files
 
   Example:
     $ beat algorithms path xxx
-  '''
-  return common.display_local_path(ctx.meta['config'].path, 'algorithm', names)
+  """
+    return common.display_local_path(ctx.meta["config"].path, "algorithm", names)
 
 
 @algorithms.command()
-@click.argument('name', nargs=1)
+@click.argument("name", nargs=1)
 @click.pass_context
 @raise_on_error
 def edit(ctx, name):
-  '''Edit local algorithm file
+    """Edit local algorithm file
 
   Example:
     $ beat algorithms edit xxx
-  '''
-  return common.edit_local_file(ctx.meta['config'].path,
-                                ctx.meta['config'].editor, 'algorithm',
-                                name)
+  """
+    return common.edit_local_file(
+        ctx.meta["config"].path, ctx.meta["config"].editor, "algorithm", name
+    )
 
 
 @algorithms.command()
-@click.argument('name', nargs=-1)
+@click.argument("name", nargs=-1)
 @click.pass_context
 @raise_on_error
 def check(ctx, name):
-  '''Checks a local algorithm for validity
+    """Checks a local algorithm for validity
 
   Example:
     $ beat algorithms check xxx
-  '''
-  return common.check(ctx.meta['config'].path, 'algorithm', name)
-
+  """
+    return common.check(ctx.meta["config"].path, "algorithm", name)
 
 
 @algorithms.command()
-@click.argument('name', nargs=-1)
-@click.option('--force', help='Performs operation regardless of conflicts',
-              is_flag=True)
+@click.argument("name", nargs=-1)
+@click.option(
+    "--force", help="Performs operation regardless of conflicts", is_flag=True
+)
 @click.pass_context
 @raise_on_error
 def pull(ctx, name, force):
-  '''Downloads the specified algorithms from the server
+    """Downloads the specified algorithms from the server
 
   Example:
     $ beat algorithms pull --force yyy
-  '''
-  with common.make_webapi(ctx.meta['config']) as webapi:
-    return pull_impl(webapi, ctx.meta['config'].path, name, force, 0, {}, {})
-
-
+  """
+    with common.make_webapi(ctx.meta["config"]) as webapi:
+        return pull_impl(webapi, ctx.meta["config"].path, name, force, 0, {}, {})
 
 
 @algorithms.command()
-@click.argument('name', nargs=-1)
-@click.option('--force', help='Performs operation regardless of conflicts',
-              is_flag=True)
-@click.option('--dry-run', help="Doesn't really perform the task, just "
-              "comments what would do", is_flag=True)
+@click.argument("name", nargs=-1)
+@click.option(
+    "--force", help="Performs operation regardless of conflicts", is_flag=True
+)
+@click.option(
+    "--dry-run",
+    help="Doesn't really perform the task, just " "comments what would do",
+    is_flag=True,
+)
 @click.pass_context
 @raise_on_error
 def push(ctx, name, force, dry_run):
-  '''Uploads algorithms to the server
+    """Uploads algorithms to the server
 
   Example:
     $ beat algorithms push --dry-run yyy
-  '''
-  with common.make_webapi(ctx.meta['config']) as webapi:
-    return common.push(webapi, ctx.meta['config'].path, 'algorithm',
-                       name, ['name', 'declaration', 'code', 'description'],
-                       {}, force, dry_run, 0)
-
+  """
+    with common.make_webapi(ctx.meta["config"]) as webapi:
+        return common.push(
+            webapi,
+            ctx.meta["config"].path,
+            "algorithm",
+            name,
+            ["name", "declaration", "code", "description"],
+            {},
+            force,
+            dry_run,
+            0,
+        )
 
 
 @algorithms.command()
-@click.argument('name', nargs=1)
+@click.argument("name", nargs=1)
 @click.pass_context
 @raise_on_error
 def diff(ctx, name):
-  '''Shows changes between the local algorithm and the remote version
+    """Shows changes between the local algorithm and the remote version
 
   Example:
     $ beat algorithms diff xxx
-  '''
-  with common.make_webapi(ctx.meta['config']) as webapi:
-    return common.diff(webapi, ctx.meta['config'].path, 'algorithm',
-                       name, ['declaration', 'code', 'description'])
-
+  """
+    with common.make_webapi(ctx.meta["config"]) as webapi:
+        return common.diff(
+            webapi,
+            ctx.meta["config"].path,
+            "algorithm",
+            name,
+            ["declaration", "code", "description"],
+        )
 
 
 @algorithms.command()
 @click.pass_context
 @raise_on_error
 def status(ctx):
-  '''Shows (editing) status for all available algorithms
+    """Shows (editing) status for all available algorithms
 
   Example:
     $ beat algorithms status
-  '''
-  with common.make_webapi(ctx.meta['config']) as webapi:
-    return common.status(webapi, ctx.meta['config'].path, 'algorithm')[0]
-
+  """
+    with common.make_webapi(ctx.meta["config"]) as webapi:
+        return common.status(webapi, ctx.meta["config"].path, "algorithm")[0]
 
 
 @algorithms.command()
-@click.argument('name', nargs=-1)
+@click.argument("name", nargs=-1)
 @click.pass_context
 @raise_on_error
 def create(ctx, name):
-  '''Creates a new local algorithm
+    """Creates a new local algorithm
 
   Example:
     $ beat algorithms create xxx
-  '''
-  return common.create(ctx.meta['config'].path, 'algorithm', name)
-
+  """
+    return common.create(ctx.meta["config"].path, "algorithm", name)
 
 
 @algorithms.command()
-@click.argument('name', nargs=1)
+@click.argument("name", nargs=1)
 @click.pass_context
 @raise_on_error
 def version(ctx, name):
-  '''Creates a new version of an existing algorithm
+    """Creates a new version of an existing algorithm
 
   Example:
     $ beat algorithms version xxx
-  '''
-  return common.new_version(ctx.meta['config'].path, 'algorithm', name)
-
+  """
+    return common.new_version(ctx.meta["config"].path, "algorithm", name)
 
 
 @algorithms.command()
-@click.argument('src', nargs=1)
-@click.argument('dst', nargs=1)
+@click.argument("src", nargs=1)
+@click.argument("dst", nargs=1)
 @click.pass_context
 @raise_on_error
 def fork(ctx, src, dst):
-  '''Forks a local algorithm
+    """Forks a local algorithm
 
   Example:
       $ beat algorithms fork xxx yyy
-  '''
-  return common.fork(ctx.meta['config'].path, 'algorithm', src, dst)
-
+  """
+    return common.fork(ctx.meta["config"].path, "algorithm", src, dst)
 
 
 @algorithms.command()
-@click.argument('name', nargs=-1)
-@click.option('--remote', help='Only acts on the remote copy of the algorithm',
-              is_flag=True)
+@click.argument("name", nargs=-1)
+@click.option(
+    "--remote", help="Only acts on the remote copy of the algorithm", is_flag=True
+)
 @click.pass_context
 @raise_on_error
 def rm(ctx, name, remote):
-  '''Deletes a local algorithm (unless --remote is specified)
+    """Deletes a local algorithm (unless --remote is specified)
 
   Example:
       $ beat algorithms rm xxx
-  '''
-  if remote:
-    with common.make_webapi(ctx.meta['config']) as webapi:
-      return common.delete_remote(webapi, 'algorithm', name)
-  else:
-    return common.delete_local(ctx.meta['config'].path, 'algorithm', name)
-
+  """
+    if remote:
+        with common.make_webapi(ctx.meta["config"]) as webapi:
+            return common.delete_remote(webapi, "algorithm", name)
+    else:
+        return common.delete_local(ctx.meta["config"].path, "algorithm", name)
 
 
 @algorithms.command()
-@click.argument('instructions', nargs=1)
-@click.option('--example', help='Display some example JSON instruction files',
-              is_flag=True)
+@click.argument("instructions", nargs=1)
+@click.option(
+    "--example", help="Display some example JSON instruction files", is_flag=True
+)
 @click.pass_context
 @raise_on_error
 def execute(ctx, instructions, example):
-  '''Execute an algorithm following instructions in a JSON file
+    """Execute an algorithm following instructions in a JSON file
 
   Example:
     $ beat algorithms execute <instructions>
     $ beat algorithms execute --examples
-  '''
-  if example:
-    print_examples()
-    return 0
+  """
+    if example:
+        print_examples()
+        return 0
 
-  return execute_impl(ctx.meta['config'].path, ctx.meta['config'].cache,
-                      instructions)
+    return execute_impl(ctx.meta["config"].path, ctx.meta["config"].cache, instructions)