From 61b56e81f0539b1742fb22ac005c5e663c76ae0d Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:50:21 +0200 Subject: [PATCH 1/9] [algorithms] Fix print statements --- beat/cmdline/algorithms.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beat/cmdline/algorithms.py b/beat/cmdline/algorithms.py index 659d877..d21c88a 100755 --- a/beat/cmdline/algorithms.py +++ b/beat/cmdline/algorithms.py @@ -169,7 +169,7 @@ def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache): def print_examples(): - print """ + print(""" To feed data from a database to an algorithm: ============================================= @@ -247,7 +247,7 @@ To execute an analyzer: "version": "" } } -""" +""") @@ -322,16 +322,16 @@ def execute(prefix, cache, instructions_file): with executor: result = executor.process() if result['status'] != 0: - print 'STDERR:' - print result['stderr'] + print('STDERR:') + print(result['stderr']) # Display the results if configuration.has_key('outputs'): # Standard algorithm - print 'Outputs of the algorithms available at:' + print('Outputs of the algorithms available at:') for name, cfg in configuration['outputs'].items(): - print ' - %s: %s' % (name, cfg['path']) + print(' - %s: %s' % (name, cfg['path'])) else: - print 'Results of the analyzer available at: %s' % configuration['result']['path'] + print('Results of the analyzer available at: %s' % configuration['result']['path']) except Exception as e: import traceback -- GitLab From 55cbe29108acb8a40b4441ee4388f9c46ecf8a93 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:50:52 +0200 Subject: [PATCH 2/9] [algorithms] Fix key search in dictionaries --- beat/cmdline/algorithms.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beat/cmdline/algorithms.py b/beat/cmdline/algorithms.py index d21c88a..4dcfea2 100755 --- a/beat/cmdline/algorithms.py +++ b/beat/cmdline/algorithms.py @@ -264,13 +264,13 @@ def execute(prefix, cache, instructions_file): configuration['queue'] = 'unused' configuration['nb_slots'] = 1 - if not configuration.has_key('parameters'): + if 'parameters' not in configuration: configuration['parameters'] = {} for name, cfg in configuration['inputs'].items(): cfg['endpoint'] = name suffix = '' - if cfg.has_key('database'): # Connected to a database output + if 'database' in cfg: # Connected to a database output cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set']) suffix = '.db' @@ -278,7 +278,7 @@ def execute(prefix, cache, instructions_file): algo = AlgorithmStorage(prefix, configuration['algorithm']) - if configuration.has_key('outputs'): # Standard algorithm + if 'outputs' in configuration: # Standard algorithm for name, cfg in configuration['outputs'].items(): cfg['endpoint'] = name cfg['hash'] = hash.hashBlockOutput( @@ -326,7 +326,7 @@ def execute(prefix, cache, instructions_file): print(result['stderr']) # Display the results - if configuration.has_key('outputs'): # Standard algorithm + if 'outputs' in configuration: # Standard algorithm print('Outputs of the algorithms available at:') for name, cfg in configuration['outputs'].items(): print(' - %s: %s' % (name, cfg['path'])) -- GitLab From 512b32a73926392aec8acd607fe47c4753ef400c Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:52:06 +0200 Subject: [PATCH 3/9] [config] Fix print statement --- beat/cmdline/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beat/cmdline/config.py b/beat/cmdline/config.py index f272a9b..e1f79f8 100644 --- a/beat/cmdline/config.py +++ b/beat/cmdline/config.py @@ -303,7 +303,7 @@ def process(args): return 0 elif args['list']: - print args['config'] + print(args['config']) return 0 elif args['set']: -- GitLab From 3ccd2569ed3b7d75b4cc023c63a08a722e85e665 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:52:23 +0200 Subject: [PATCH 4/9] [config] Fix open mode syntax --- beat/cmdline/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beat/cmdline/config.py b/beat/cmdline/config.py index e1f79f8..37ac708 100644 --- a/beat/cmdline/config.py +++ b/beat/cmdline/config.py @@ -254,7 +254,7 @@ class Configuration(object): c = self.__config_file() dirname = os.path.dirname(c) if not os.path.exists(dirname): os.makedirs(dirname) - with os.fdopen(os.open(c, os.O_WRONLY | os.O_CREAT, 0600), 'wt') as f: + with os.fdopen(os.open(c, os.O_WRONLY | os.O_CREAT, 0o600), 'wt') as f: simplejson.dump(self.__data, f, indent=4) def _is_valid_key(self, key): -- GitLab From f64e770c52c01a9c65c158eb263983b3aab9fdbd Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:52:49 +0200 Subject: [PATCH 5/9] [databases] Fix print statements --- beat/cmdline/databases.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beat/cmdline/databases.py b/beat/cmdline/databases.py index 1d8d463..b1666ea 100755 --- a/beat/cmdline/databases.py +++ b/beat/cmdline/databases.py @@ -582,7 +582,7 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None, if start != previous_start: print(80 * '-') - print 'FROM %d TO %d' % (start, end) + print('FROM %d TO %d' % (start, end)) whole_inputs = [input_ for input_ in input_group if input_.data_index == start and @@ -590,7 +590,7 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None, for input in whole_inputs: label = ' - ' + str(input.name) + ': ' - print label + data_to_json(input.data, len(label)) + print(label + data_to_json(input.data, len(label))) previous_start = start @@ -612,11 +612,11 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None, for key in sorted_keys: print - print ' FROM %d TO %d' % key + print(' FROM %d TO %d' % key) for input in grouped_inputs[key]: label = ' - ' + str(input.name) + ': ' - print label + data_to_json(input.data, len(label)) + print(label + data_to_json(input.data, len(label))) except Exception as e: logger.error("Failed to retrieve the next data: %s", e) -- GitLab From b341101c9021c018895fa59f56074d215a3d0b01 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:53:19 +0200 Subject: [PATCH 6/9] [databases] Fix sorting statement --- beat/cmdline/databases.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beat/cmdline/databases.py b/beat/cmdline/databases.py index b1666ea..499d7d7 100755 --- a/beat/cmdline/databases.py +++ b/beat/cmdline/databases.py @@ -607,8 +607,7 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None, grouped_inputs[key] = [] grouped_inputs[key].append(input) - sorted_keys = grouped_inputs.keys() - sorted_keys.sort() + sorted_keys = sorted(grouped_inputs.keys()) for key in sorted_keys: print -- GitLab From 63a576a3b01a1c5c6ffa7dfdee93506316e271e6 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:53:36 +0200 Subject: [PATCH 7/9] [test] Fix exception list statement --- beat/cmdline/test/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beat/cmdline/test/__init__.py b/beat/cmdline/test/__init__.py index 7fff177..f0c2016 100644 --- a/beat/cmdline/test/__init__.py +++ b/beat/cmdline/test/__init__.py @@ -63,7 +63,7 @@ if platform: try: code = urllib.request.urlopen(platform).getcode() disconnected = code != 200 - except IOError, urllib.URLError: + except (IOError, urllib.URLError): disconnected = True else: platform = 'User did not set $BEAT_CMDLINE_TEST_PLATFORM' -- GitLab From 07dc01a231419fd91e6b6e2fc24ad43afa17c918 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:54:38 +0200 Subject: [PATCH 8/9] [common] Fix diff parameters Don't use python keywords as variable name. --- beat/cmdline/common.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beat/cmdline/common.py b/beat/cmdline/common.py index 17ba5d7..97bb49f 100644 --- a/beat/cmdline/common.py +++ b/beat/cmdline/common.py @@ -696,7 +696,7 @@ def pull(webapi, prefix, type, names, fields, force, indentation): return status, list(available) -def diff(webapi, prefix, type, name, fields): +def diff(webapi, prefix, type_, name, fields): """Shows the differences between two objects, for each of the fields @@ -737,8 +737,8 @@ def diff(webapi, prefix, type, name, fields): def _eval_diff(remote, local, ext): '''Calculates differences between two string buffers''' return difflib.unified_diff(remote.split('\n'), local.split('\n'), - os.path.join('remote', type, name + ext), - os.path.join('local', type, name + ext)) + os.path.join('remote', type_, name + ext), + os.path.join('local', type_, name + ext)) def _show_diff(diffs): @@ -750,9 +750,9 @@ def diff(webapi, prefix, type, name, fields): else: print(line) - storage = TYPE_STORAGE[type](prefix, name) + storage = TYPE_STORAGE[type_](prefix, name) local = storage.load() #may also return a tuple, depending on the type - remote = fetch_object(webapi, type, name, fields) + remote = fetch_object(webapi, type_, name, fields) if remote is None: return 1 if 'declaration' in remote and \ not isinstance(remote['declaration'], six.string_types): @@ -767,12 +767,12 @@ def diff(webapi, prefix, type, name, fields): for field in fields: diffs = _eval_diff(remote[field], local[field], extension[field]) if diffs: - logger.info("differences for `%s' of `%s/%s':", field, TYPE_PLURAL[type], + logger.info("differences for `%s' of `%s/%s':", field, TYPE_PLURAL[type_], name) _show_diff(diffs) else: logger.info("no differences for `%s' of `%s/%s'", field, - TYPE_PLURAL[type], name) + TYPE_PLURAL[type_], name) return 0 -- GitLab From 45a75e518bbfd3e0966db360bd4b160a90430d02 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 26 Apr 2018 08:55:37 +0200 Subject: [PATCH 9/9] [common] Decode inputs before feeding to difflib --- beat/cmdline/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beat/cmdline/common.py b/beat/cmdline/common.py index 97bb49f..e4fbc99 100644 --- a/beat/cmdline/common.py +++ b/beat/cmdline/common.py @@ -736,6 +736,12 @@ def diff(webapi, prefix, type_, name, fields): def _eval_diff(remote, local, ext): '''Calculates differences between two string buffers''' + + if not isinstance(local, six.string_types): + local = local.decode('utf-8') + if not isinstance(remote, six.string_types): + remote = remote.decode('utf-8') + return difflib.unified_diff(remote.split('\n'), local.split('\n'), os.path.join('remote', type_, name + ext), os.path.join('local', type_, name + ext)) -- GitLab