Commit 6c8da8c3 authored by Flavio TARSETTI's avatar Flavio TARSETTI
Browse files

Merge branch '60_add_support_protocol_template' into 'master'

Add support protocol template

See merge request !66
parents 98de40d8 693a8030
Pipeline #29873 passed with stages
in 7 minutes and 33 seconds
### Bandit config file generated from:
# '/home/sgaist/miniconda3/envs/test_bandit/bin/bandit-config-generator -s B610 -o .bandit'
### This config may optionally select a subset of tests to run or skip by
### filling out the 'tests' and 'skips' lists given below. If no tests are
### specified for inclusion then it is assumed all tests are desired. The skips
### set will remove specific tests from the include set. This can be controlled
### using the -t/-s CLI options. Note that the same test ID should not appear
### in both 'tests' and 'skips', this would be nonsensical and is detected by
### Bandit at runtime.
# Available tests:
# B101 : assert_used
# B102 : exec_used
# B103 : set_bad_file_permissions
# B104 : hardcoded_bind_all_interfaces
# B105 : hardcoded_password_string
# B106 : hardcoded_password_funcarg
# B107 : hardcoded_password_default
# B108 : hardcoded_tmp_directory
# B110 : try_except_pass
# B112 : try_except_continue
# B201 : flask_debug_true
# B301 : pickle
# B302 : marshal
# B303 : md5
# B304 : ciphers
# B305 : cipher_modes
# B306 : mktemp_q
# B307 : eval
# B308 : mark_safe
# B309 : httpsconnection
# B310 : urllib_urlopen
# B311 : random
# B312 : telnetlib
# B313 : xml_bad_cElementTree
# B314 : xml_bad_ElementTree
# B315 : xml_bad_expatreader
# B316 : xml_bad_expatbuilder
# B317 : xml_bad_sax
# B318 : xml_bad_minidom
# B319 : xml_bad_pulldom
# B320 : xml_bad_etree
# B321 : ftplib
# B322 : input
# B323 : unverified_context
# B324 : hashlib_new_insecure_functions
# B325 : tempnam
# B401 : import_telnetlib
# B402 : import_ftplib
# B403 : import_pickle
# B404 : import_subprocess
# B405 : import_xml_etree
# B406 : import_xml_sax
# B407 : import_xml_expat
# B408 : import_xml_minidom
# B409 : import_xml_pulldom
# B410 : import_lxml
# B411 : import_xmlrpclib
# B412 : import_httpoxy
# B413 : import_pycrypto
# B414 : import_pycryptodome
# B501 : request_with_no_cert_validation
# B502 : ssl_with_bad_version
# B503 : ssl_with_bad_defaults
# B504 : ssl_with_no_version
# B505 : weak_cryptographic_key
# B506 : yaml_load
# B507 : ssh_no_host_key_verification
# B601 : paramiko_calls
# B602 : subprocess_popen_with_shell_equals_true
# B603 : subprocess_without_shell_equals_true
# B604 : any_other_function_with_shell_equals_true
# B605 : start_process_with_a_shell
# B606 : start_process_with_no_shell
# B607 : start_process_with_partial_path
# B608 : hardcoded_sql_expressions
# B609 : linux_commands_wildcard_injection
# B610 : django_extra_used
# B611 : django_rawsql_used
# B701 : jinja2_autoescape_false
# B702 : use_of_mako_templates
# B703 : django_mark_safe
# (optional) list included test IDs here, eg '[B101, B406]':
tests:
# (optional) list skipped test IDs here, eg '[B101, B406]':
skips: ['B610']
### (optional) plugin settings - some test plugins require configuration data
### that may be given here, per-plugin. All bandit test plugins have a built in
### set of sensible defaults and these will be used if no configuration is
### provided. It is not necessary to provide settings for every (or any) plugin
### if the defaults are acceptable.
any_other_function_with_shell_equals_true:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
hardcoded_tmp_directory:
tmp_dirs:
- /tmp
- /var/tmp
- /dev/shm
linux_commands_wildcard_injection:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
ssl_with_bad_defaults:
bad_protocol_versions:
- PROTOCOL_SSLv2
- SSLv2_METHOD
- SSLv23_METHOD
- PROTOCOL_SSLv3
- PROTOCOL_TLSv1
- SSLv3_METHOD
- TLSv1_METHOD
ssl_with_bad_version:
bad_protocol_versions:
- PROTOCOL_SSLv2
- SSLv2_METHOD
- SSLv23_METHOD
- PROTOCOL_SSLv3
- PROTOCOL_TLSv1
- SSLv3_METHOD
- TLSv1_METHOD
start_process_with_a_shell:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
start_process_with_no_shell:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
start_process_with_partial_path:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
subprocess_popen_with_shell_equals_true:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
subprocess_without_shell_equals_true:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
try_except_continue:
check_typed_exception: false
try_except_pass:
check_typed_exception: false
weak_cryptographic_key:
weak_key_size_dsa_high: 1024
weak_key_size_dsa_medium: 2048
weak_key_size_ec_high: 160
weak_key_size_ec_medium: 224
weak_key_size_rsa_high: 1024
weak_key_size_rsa_medium: 2048
......@@ -2,6 +2,7 @@
*.swp
*.pyc
*.so
.beatrc
bin
eggs
parts
......
......@@ -8,22 +8,21 @@ repos:
language_version: python3.6
exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.0.0
rev: v2.2.1
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: debug-statements
exclude: beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
- id: check-added-large-files
- id: check-docstring-first
- id: flake8
- id: check-yaml
exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/bandit
rev: 'master' # Update me!
rev: master # Update me!
hooks:
- id: bandit
exclude: beat/editor/test|beat/core/test/prefix/algorithms/user/syntax_error/1.py|beat/core/test/prefix/databases/invalid/1.py
args: [-c, .bandit.yml]
- repo: local
hooks:
- id: sphinx-build
......
......@@ -47,6 +47,8 @@ from beat.core import hash
from beat.backend.python.algorithm import Storage as AlgorithmStorage
from . import common
from . import commands
from .decorators import raise_on_error
from .click_helper import AliasedGroup
......@@ -55,7 +57,7 @@ logger = logging.getLogger(__name__)
def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache):
"""Copies algorithms (and required libraries/dataformats) from the server.
"""Copies algorithms (and required libraries/dataformats) from the server.
Parameters:
......@@ -93,37 +95,48 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache
"""
from .dataformats import pull_impl as dataformats_pull
from .libraries import pull_impl as libraries_pull
status, names = common.pull(webapi, prefix, 'algorithm', names,
['declaration', 'code', 'description'], force, indentation)
from .dataformats import pull_impl as dataformats_pull
from .libraries import pull_impl as libraries_pull
if status != 0: return status
status, names = common.pull(
webapi,
prefix,
"algorithm",
names,
["declaration", "code", "description"],
force,
indentation,
)
# see what dataformats one needs to pull
indent = indentation * ' '
if status != 0:
return status
dataformats = []
libraries = []
for name in names:
obj = algorithm.Algorithm(prefix, name)
dataformats.extend(obj.dataformats.keys())
libraries.extend(obj.libraries.keys())
# see what dataformats one needs to pull
if indentation == 0:
indentation = 4
# downloads any formats to which we depend on
df_status = dataformats_pull(webapi, prefix, dataformats, force,
indentation + 2, format_cache)
dataformats = []
libraries = []
for name in names:
obj = algorithm.Algorithm(prefix, name)
dataformats.extend(obj.dataformats.keys())
libraries.extend(obj.libraries.keys())
lib_status = libraries_pull(webapi, prefix, libraries, force,
indentation + 2, lib_cache)
# downloads any formats to which we depend on
df_status = dataformats_pull(
webapi, prefix, dataformats, force, indentation, format_cache
)
return status + df_status + lib_status
lib_status = libraries_pull(
webapi, prefix, libraries, force, indentation, lib_cache
)
return status + df_status + lib_status
def print_examples():
print("""
print(
"""
To feed data from a database to an algorithm:
=============================================
......@@ -201,314 +214,207 @@ To execute an analyzer:
"version": "<environment_version>"
}
}
""")
"""
)
def execute_impl(prefix, cache, instructions_file):
try:
# Load the JSON configuration
if not os.path.exists(instructions_file):
raise IOError("JSON instructions file `%s' not found" % instructions_file)
with open(instructions_file, 'r') as f:
configuration = json.load(f)
# Add missing configuration fields
configuration['queue'] = 'unused'
configuration['nb_slots'] = 1
if 'parameters' not in configuration:
configuration['parameters'] = {}
for name, cfg in configuration['inputs'].items():
cfg['endpoint'] = name
suffix = ''
if 'database' in cfg: # Connected to a database output
cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set'])
suffix = '.db'
cfg['path'] = hash.toPath(cfg['hash'], suffix=suffix)
algo = AlgorithmStorage(prefix, configuration['algorithm'])
if 'outputs' in configuration: # Standard algorithm
for name, cfg in configuration['outputs'].items():
cfg['endpoint'] = name
cfg['hash'] = hash.hashBlockOutput(
'block',
configuration['algorithm'],
algo.hash(),
configuration['parameters'],
configuration['environment'],
dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
name,
)
cfg['path'] = hash.toPath(cfg['hash'], '')
else: # Analyzer
configuration['result'] = {}
configuration['result']['hash'] = hash.hashAnalyzer(
'block',
configuration['algorithm'],
algo.hash(),
configuration['parameters'],
configuration['environment'],
dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
)
configuration['result']['path'] = hash.toPath(configuration['result']['hash'], '')
# Sets up the execution
dataformat_cache = {}
database_cache = {}
algorithm_cache = {}
host = Host(raise_on_errors=False)
executor = DockerExecutor(host, prefix, configuration, cache, dataformat_cache,
database_cache, algorithm_cache)
if not executor.valid:
logger.error('Invalid configuration:\n * %s' % '\n * '.join(executor.errors))
return 1
# Execute the algorithm
with executor:
result = executor.process()
if result['status'] != 0:
print('STDERR:')
print(result['stderr'])
# Display the results
if 'outputs' in configuration: # Standard algorithm
print('Outputs of the algorithms available at:')
for name, cfg in configuration['outputs'].items():
print(' - %s: %s' % (name, cfg['path']))
else:
print('Results of the analyzer available at: %s' % configuration['result']['path'])
except Exception as e:
import traceback
logger.error(traceback.format_exc())
return 1
return 0
try:
# Load the JSON configuration
if not os.path.exists(instructions_file):
raise IOError("JSON instructions file `%s' not found" % instructions_file)
with open(instructions_file, "r") as f:
configuration = json.load(f)
# Add missing configuration fields
configuration["queue"] = "unused"
configuration["nb_slots"] = 1
if "parameters" not in configuration:
configuration["parameters"] = {}
for name, cfg in configuration["inputs"].items():
cfg["endpoint"] = name