algorithms.py 12.9 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.cmdline module of the BEAT platform.          #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


"""Usage:
  %(prog)s algorithms list [--remote]
  %(prog)s algorithms check [<name>]...
  %(prog)s algorithms pull [--force] [<name>]...
  %(prog)s algorithms push [--force] [--dry-run] [<name>]...
  %(prog)s algorithms diff <name>
  %(prog)s algorithms status
  %(prog)s algorithms create <name>...
  %(prog)s algorithms version <name>
  %(prog)s algorithms fork <src> <dst>
  %(prog)s algorithms rm [--remote] <name>...
40
41
  %(prog)s algorithms execute <instructions>
  %(prog)s algorithms execute --examples
André Anjos's avatar
André Anjos committed
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  %(prog)s algorithms --help


Commands:
  list      Lists all the algorithms available on the platform
  check     Checks a local algorithm for validity
  pull      Downloads the specified algorithms from the server
  push      Uploads algorithms to the server
  diff      Shows changes between the local algorithm and the remote version
  status    Shows (editing) status for all available algorithms
  create    Creates a new local algorithm
  version   Creates a new version of an existing algorithm
  fork      Forks a local algorithm
  rm        Deletes a local algorithm (unless --remote is specified)
56
  execute   Execute an algorithm following instructions in a JSON file
André Anjos's avatar
André Anjos committed
57
58
59


Options:
60
61
62
63
64
65
66
67
68
  --force     Performs operation regardless of conflicts
  --dry-run   Doesn't really perform the task, just comments what would do
  --remote    Only acts on the remote copy of the algorithm
  --examples  Display some example JSON instruction files
  --help      Display this screen


Arguments for 'execute':
  <instructions>  JSON file containing the instructions
André Anjos's avatar
André Anjos committed
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85


Examples:

  To list all algorithms available locally:

    $ %(prog)s al list

  To list all algorithms available at the platform:

    $ %(prog)s al list --remote

"""

import logging
logger = logging.getLogger(__name__)

86
87
88
89
90
import os
import sys
import docopt
import simplejson as json

André Anjos's avatar
André Anjos committed
91
92
93
from . import common

from beat.core import algorithm
94
from beat.core.execution import DockerExecutor
95
96
97
98
99
100
from beat.core.dock import Host
from beat.core import hash

from beat.backend.python.database import Storage as DatabaseStorage
from beat.backend.python.algorithm import Storage as AlgorithmStorage

André Anjos's avatar
André Anjos committed
101
102
103
104
105
106
107
108
109
110
111
112
113


def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
  """Copies algorithms (and required libraries/dataformats) from the server.

  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest

    prefix (str): A string representing the root of the path in which the user
      objects are stored

André Anjos's avatar
André Anjos committed
114
115
116
117
118
    names (:py:class:`list`): A list of strings, each representing the unique
      relative path of the objects to retrieve or a list of usernames from
      which to retrieve objects. If the list is empty, then we pull all
      available objects of a given type. If no user is set, then pull all
      public objects of a given type.
André Anjos's avatar
André Anjos committed
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

    force (bool): If set to ``True``, then overwrites local changes with the
      remotely retrieved copies.

    indentation (int): The indentation level, useful if this function is called
      recursively while downloading different object types. This is normally
      set to ``0`` (zero).

    format_cache (dict): A dictionary containing all dataformats already
      downloaded.

    lib_cache (dict): A dictionary containing all libraries already
      downloaded.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

  """

  from .dataformats import pull as dataformats_pull
  from .libraries import pull as libraries_pull

  status, names = common.pull(webapi, prefix, 'algorithm', names,
Philip ABBET's avatar
Philip ABBET committed
146
                              ['declaration', 'code', 'description'], force, indentation)
André Anjos's avatar
André Anjos committed
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

  if status != 0: return status

  # see what dataformats one needs to pull
  indent = indentation * ' '

  dataformats = []
  libraries = []
  for name in names:
    obj = algorithm.Algorithm(prefix, name)
    dataformats.extend(obj.dataformats.keys())
    libraries.extend(obj.libraries.keys())

  # downloads any formats to which we depend on
  df_status = dataformats_pull(webapi, prefix, dataformats, force,
Philip ABBET's avatar
Philip ABBET committed
162
                               indentation + 2, format_cache)
André Anjos's avatar
André Anjos committed
163
164

  lib_status = libraries_pull(webapi, prefix, libraries, force,
Philip ABBET's avatar
Philip ABBET committed
165
                              indentation + 2, lib_cache)
André Anjos's avatar
André Anjos committed
166
167
168
169

  return status + df_status + lib_status


170
171

def print_examples():
172
    print("""
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
To feed data from a database to an algorithm:
=============================================

{
    "algorithm": "<username>/<algorithm>/<version>",
    "channel": "main",
    "inputs": {
        "<input_name>": {
            "database": "<database>/<version>",
            "protocol": "<protocol>",
            "set": "<set>",
            "output": "<output_name>",
            "channel": "main"
        }
    },
    "outputs": {
        "<output_name>": {
Philip ABBET's avatar
Philip ABBET committed
190
          "channel": "main"
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
        }
    },
    "parameters": {
      "<parameter_name>": <value>
    },
    "environment": {
        "name": "<environment_name>",
        "version": "<environment_version>"
    }
}


To feed data from a file in the cache:
======================================

{
    "algorithm": "<username>/<algorithm>/<version>",
    "channel": "main",
    "inputs": {
        "<input_name>": {
            "hash": "<hash>",
            "channel": "main"
        }
    },
    "outputs": {
        "<output_name>": {
Philip ABBET's avatar
Philip ABBET committed
217
          "channel": "main"
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
        }
    },
    "parameters": {
      "<parameter_name>": <value>
    },
    "environment": {
        "name": "<environment_name>",
        "version": "<environment_version>"
    }
}


To execute an analyzer:
=======================

{
    "algorithm": "<username>/<algorithm>/<version>",
    "channel": "main",
    "inputs": {
        "<input_name>": {
            "hash": "<hash>",
            "channel": "main"
        }
    },
    "parameters": {
      "<parameter_name>": <value>
    },
    "environment": {
        "name": "<environment_name>",
        "version": "<environment_version>"
    }
}
250
""")
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266



def execute(prefix, cache, instructions_file):
  try:
    # Load the JSON configuration
    if not os.path.exists(instructions_file):
      raise IOError("JSON instructions file `%s' not found" % instructions_file)

    with open(instructions_file, 'r') as f:
      configuration = json.load(f)

    # Add missing configuration fields
    configuration['queue'] = 'unused'
    configuration['nb_slots'] = 1

267
    if 'parameters' not in configuration:
268
269
270
271
      configuration['parameters'] = {}

    for name, cfg in configuration['inputs'].items():
      cfg['endpoint'] = name
272
      suffix = ''
273
      if 'database' in cfg:   # Connected to a database output
274
275
276
277
        cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set'])
        suffix = '.db'

      cfg['path'] = hash.toPath(cfg['hash'], suffix=suffix)
278
279
280

    algo = AlgorithmStorage(prefix, configuration['algorithm'])

281
    if 'outputs' in  configuration:   # Standard algorithm
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
      for name, cfg in configuration['outputs'].items():
          cfg['endpoint'] = name
          cfg['hash'] = hash.hashBlockOutput(
              'block',
              configuration['algorithm'],
              algo.hash(),
              configuration['parameters'],
              configuration['environment'],
              dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
              name,
          )
          cfg['path'] = hash.toPath(cfg['hash'], '')

    else:   # Analyzer
      configuration['result'] = {}
      configuration['result']['hash'] = hash.hashAnalyzer(
Philip ABBET's avatar
Philip ABBET committed
298
          'block',
299
300
301
302
303
              configuration['algorithm'],
              algo.hash(),
              configuration['parameters'],
              configuration['environment'],
              dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
Philip ABBET's avatar
Philip ABBET committed
304
      )
305
306
307
308
309
310
311
      configuration['result']['path'] = hash.toPath(configuration['result']['hash'], '')

    # Sets up the execution
    dataformat_cache = {}
    database_cache = {}
    algorithm_cache = {}

Philip ABBET's avatar
Philip ABBET committed
312
    host = Host(raise_on_errors=False)
313

314
315
    executor = DockerExecutor(host, prefix, configuration, cache, dataformat_cache,
                              database_cache, algorithm_cache)
316
317
318
319
320
321
322

    if not executor.valid:
      logger.error('Invalid configuration:\n  * %s' % '\n  * '.join(executor.errors))
      return 1

    # Execute the algorithm
    with executor:
Philip ABBET's avatar
Philip ABBET committed
323
      result = executor.process()
324
      if result['status'] != 0:
325
326
          print('STDERR:')
          print(result['stderr'])
327
328

    # Display the results
329
    if 'outputs' in configuration:   # Standard algorithm
330
      print('Outputs of the algorithms available at:')
331
      for name, cfg in configuration['outputs'].items():
332
        print('  - %s: %s' % (name, cfg['path']))
333
    else:
334
      print('Results of the analyzer available at: %s' % configuration['result']['path'])
335
336
337
338
339
340
341
342
343
344

  except Exception as e:
    import traceback
    logger.error(traceback.format_exc())
    return 1

  return 0



André Anjos's avatar
André Anjos committed
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
def process(args):

  if args['list']:
    if args['--remote']:
      with common.make_webapi(args['config']) as webapi:
        return common.display_remote_list(webapi, 'algorithm')
    else:
      return common.display_local_list(args['config'].path, 'algorithm')

  elif args['check']:
    return common.check(args['config'].path, 'algorithm', args['<name>'])

  elif args['pull']:
    with common.make_webapi(args['config']) as webapi:
      return pull(webapi, args['config'].path, args['<name>'],
Philip ABBET's avatar
Philip ABBET committed
360
                  args['--force'], 0, {}, {})
André Anjos's avatar
André Anjos committed
361
362
363
364

  elif args['push']:
    with common.make_webapi(args['config']) as webapi:
      return common.push(webapi, args['config'].path, 'algorithm',
Philip ABBET's avatar
Philip ABBET committed
365
366
                         args['<name>'], ['name', 'declaration', 'code', 'description'],
                         {}, args['--force'], args['--dry-run'], 0)
André Anjos's avatar
André Anjos committed
367
368
369
370

  elif args['diff']:
    with common.make_webapi(args['config']) as webapi:
      return common.diff(webapi, args['config'].path, 'algorithm',
Philip ABBET's avatar
Philip ABBET committed
371
                         args['<name>'][0], ['declaration', 'code', 'description'])
André Anjos's avatar
André Anjos committed
372
373
374
375
376
377
378
379
380
381

  elif args['status']:
    with common.make_webapi(args['config']) as webapi:
      return common.status(webapi, args['config'].path, 'algorithm')[0]

  elif args['create']:
    return common.create(args['config'].path, 'algorithm', args['<name>'])

  elif args['version']:
    return common.new_version(args['config'].path, 'algorithm',
Philip ABBET's avatar
Philip ABBET committed
382
                              args['<name>'][0])
André Anjos's avatar
André Anjos committed
383
384
385

  elif args['fork']:
    return common.fork(args['config'].path, 'algorithm',
Philip ABBET's avatar
Philip ABBET committed
386
                       args['<src>'], args['<dst>'])
André Anjos's avatar
André Anjos committed
387

388
389
390
391
392
393
394
  elif args['execute']:
    if args['--examples']:
      print_examples()
      return 0

    return execute(args['config'].path, args['config'].cache, args['<instructions>'])

André Anjos's avatar
André Anjos committed
395
396
397
398
399
400
  elif args['rm']:
    if args['--remote']:
      with common.make_webapi(args['config']) as webapi:
        return common.delete_remote(webapi, 'algorithm', args['<name>'])
    else:
      return common.delete_local(args['config'].path, 'algorithm',
Philip ABBET's avatar
Philip ABBET committed
401
                                 args['<name>'])
André Anjos's avatar
André Anjos committed
402
403
404
405

  # Should not happen
  logger.error("unrecognized `algorithms' subcommand")
  return 1