Commit 0f0c0d01 authored by Samuel GAIST's avatar Samuel GAIST
Browse files

Merge branch 'plotter_plotterparameters' into '1.4.x'

Plotter plotterparameters and experiments plotting

See merge request !14
parents 639fd08d 6881ef81
Pipeline #19918 passed with stages
in 44 minutes and 42 seconds
......@@ -57,6 +57,7 @@ TYPE_GLOB = {
'library': os.path.join('*', '*', '*.json'),
'algorithm': os.path.join('*', '*', '*.json'),
'plotter': os.path.join('*', '*', '*.json'),
'plotterparameter': os.path.join('*', '*', '*.json'),
'toolchain': os.path.join('*', '*', '*.json'),
'experiment': os.path.join('*', '*', '*', '*', '*.json'),
}
......@@ -68,6 +69,7 @@ TYPE_FNMATCH = {
'library': os.path.splitext(TYPE_GLOB['library'])[0],
'algorithm': os.path.splitext(TYPE_GLOB['algorithm'])[0],
'plotter': os.path.splitext(TYPE_GLOB['plotter'])[0],
'plotterparameter': os.path.splitext(TYPE_GLOB['plotterparameter'])[0],
'toolchain': os.path.splitext(TYPE_GLOB['toolchain'])[0],
'experiment': os.path.splitext(TYPE_GLOB['experiment'])[0],
}
......@@ -93,12 +95,20 @@ TYPE_STORAGE = {
'experiment': experiment.Storage,
}
NOSTORAGE = ['plotterparameter']
TYPE_NOSTORAGE = {
'plotterparameter': 'plotterparameters',
}
TYPE_PLURAL = {
'dataformat': 'dataformats',
'database': 'databases',
'library': 'libraries',
'algorithm': 'algorithms',
'plotter': 'plotters',
'plotterparameter': 'plotters/plotterparameters',
'defaultplotter': 'plotters/defaultplotters',
'toolchain': 'toolchains',
'experiment': 'experiments',
}
......@@ -611,7 +621,10 @@ def fetch_object(webapi, type, name, fields):
"""
fields = '?object_format=string&fields=%s' % ','.join(fields)
url = '/api/v1/%s/%s/%s' % (TYPE_PLURAL[type], name, fields)
if name is not None:
url = '/api/v1/%s/%s/%s' % (TYPE_PLURAL[type], name, fields)
else:
url = '/api/v1/%s/%s' % (TYPE_PLURAL[type], fields)
(status_code, content) = webapi.get(url)
if status_code is None: return None
......@@ -681,20 +694,63 @@ def pull(webapi, prefix, type, names, fields, force, indentation):
status = 0
for name in names:
storage = TYPE_STORAGE[type](prefix, name)
if storage.exists() and not force: #exists locally, force not set
logger.extra("%sskipping download of `%s/%s' (exists locally)", indent,
TYPE_PLURAL[type], name)
available.add(name)
continue
if type not in NOSTORAGE:
# typical workflow with normal storage
storage = TYPE_STORAGE[type](prefix, name)
if storage.exists() and not force: #exists locally, force not set
logger.extra("%sskipping download of `%s/%s' (exists locally)", indent,
TYPE_PLURAL[type], name)
available.add(name)
continue
else:
logger.info("%sretrieving `%s/%s'...", indent, TYPE_PLURAL[type], name)
data = fetch_object(webapi, type, name, fields)
if data is None:
status += 1 #error
continue
storage.save(**data)
available.add(name)
else:
logger.info("%sretrieving `%s/%s'...", indent, TYPE_PLURAL[type], name)
data = fetch_object(webapi, type, name, fields)
if data is None:
status += 1 #error
# other workflow with no storage (i.e.: plotterparameter)
# check if storage exists
path = os.path.join(prefix, TYPE_NOSTORAGE[type], name.rsplit('/', 1)[0])
if os.path.exists(path) and not force:
logger.extra("%sskipping download of `%s/%s' (exists locally)", indent,
TYPE_PLURAL[type], name)
available.add(name)
continue
storage.save(**data)
available.add(name)
else:
# create folder
d = os.path.dirname(path)
if not os.path.exists(path):
os.makedirs(path)
logger.info("%sretrieving `%s/%s'...", indent, TYPE_PLURAL[type], name)
data = fetch_object(webapi, type, name, fields)
if data is None:
status += 1 #error
continue
from beat.backend.python import utils
if 'data' in fields:
name_path = os.path.join(prefix, TYPE_NOSTORAGE[type], name)
data_json = utils.File(name_path + '.json')
_buf_data = {}
if type is 'plotterparameter':
# generate plotterparameter data
_buf_data['plotter'] = data['plotter']
_buf_data['data'] = data['data']
_buf_data['description'] = data['short_description']
_data = simplejson.dumps(_buf_data, indent=4)
else:
_buf_data['data'] = data['data']
_data = simplejson.dumps(_buf_data['data'], indent=4)
data_json.save(_data)
if 'short_description' in fields:
name_path = os.path.join(prefix, TYPE_NOSTORAGE[type], name)
data_doc = utils.File(name_path + '.rst')
data_doc.save(data['short_description'].encode('utf8'))
available.add(name)
return status, list(available)
......
......@@ -38,6 +38,7 @@
%(prog)s experiments fork <src> <dst>
%(prog)s experiments rm [--remote] <name>...
%(prog)s experiments draw [--path=<dir>] [<name>]...
%(prog)s experiments plot [--force] [--remote] [--outputfolder=<folder>][<name>]...
%(prog)s experiments --help
......@@ -88,6 +89,8 @@ from beat.core.dock import Host
from beat.core.hash import toPath
from beat.core.hash import hashDataset
from .plotters import plot as plotters_plot
from .plotters import pull as plotters_pull
logger = logging.getLogger(__name__)
......@@ -426,6 +429,130 @@ def pull(webapi, prefix, names, force, indentation, format_cache):
return status + tc_status + db_status + algo_status
def plot(webapi, configuration, prefix, names, remote_results, force, indentation, format_cache, outputfolder=None):
"""Plots experiments from the server.
Parameters:
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
configuration (object): An instance of the configuration, to access the
BEAT server and current configuration for information
prefix (str): A string representing the root of the path in which the
user objects are stored
names (:py:class:`list`): A list of strings, each representing the unique relative
path of the objects to retrieve or a list of usernames from which to
retrieve objects. If the list is empty, then we pull all available
objects of a given type. If no user is set, then pull all public
objects of a given type.
remote_results(bool): If set to ``True``, then fetch results data
for the experiments from the server.
force (bool): If set to ``True``, then overwrites local changes with the
remotely retrieved copies.
indentation (int): The indentation level, useful if this function is
called recursively while downloading different object types. This is
normally set to ``0`` (zero).
outputfolder (str): A string representing the path in which the
experiments plot will be stored
Returns:
int: Indicating the exit status of the command, to be reported back to
the calling process. This value should be zero if everything works OK,
otherwise, different than zero (POSIX compliance).
"""
status = 0
RESULTS_SIMPLE_TYPE_NAMES = ('int32', 'float32', 'bool', 'string')
if remote_results:
if outputfolder is None:
output_folder = configuration.path
else:
# check if directory exists else create
if not os.path.isdir(outputfolder):
os.mkdir(os.path.join(configuration.path, outputfolder))
output_folder = os.path.join(configuration.path, outputfolder)
for name in names:
if not remote_results:
if outputfolder is None:
output_folder = os.path.join(configuration.path, common.TYPE_PLURAL['experiment'], name.rsplit('/', 1)[0])
else:
# check if directory exists else create
if not os.path.isdir(outputfolder):
os.mkdir(os.path.join(configuration.path, outputfolder))
output_folder = os.path.join(configuration.path, outputfolder)
if not os.path.exists(configuration.cache) or remote_results:
experiment = simplejson.loads(simplejson.dumps(common.fetch_object(webapi, "experiment", name, ['results'])))
results = experiment['results']['analysis']
for key, value in results.iteritems():
# remove non plottable results
if value['type'] not in RESULTS_SIMPLE_TYPE_NAMES:
output_name = key + '.png'
output_name = os.path.join(output_folder, output_name)
pl_status = plotters_pull(webapi, configuration.path, [value['type']], force, indentation + 2, {})
plot_status = plotters_plot(webapi, configuration.path, [value['type']], False, False, value['value'],
output_name, None, indentation + 2, format_cache)
status += pl_status
status += plot_status
else:
# make sure experiment exists locally or pull it
experiments = pull(webapi, configuration.path, [name], force, indentation, format_cache)
# get information from cache
dataformat_cache = {}
database_cache = {}
algorithm_cache = {}
library_cache = {}
experiment = Experiment(configuration.path, name,
dataformat_cache, database_cache,
algorithm_cache, library_cache)
scheduled = experiment.setup()
for key, value in scheduled.items():
executor = LocalExecutor(configuration.path,
value['configuration'],
configuration.cache, dataformat_cache,
database_cache, algorithm_cache,
library_cache,
configuration.database_paths)
if 'result' in executor.data:
f = CachedDataSource()
assert f.setup(os.path.join(executor.cache,
executor.data['result']['path'] + '.data'),
executor.prefix)
data, start, end = f[0]
for the_data in data.as_dict():
attr = getattr(data, the_data)
if attr.__class__.__name__.startswith('plot'):
datatype = attr.__class__.__name__.replace('_','/')
# remove non plottable results
if datatype not in RESULTS_SIMPLE_TYPE_NAMES:
output_name = the_data + '.png'
output_name = os.path.join(output_folder, output_name)
pl_status = plotters_pull(webapi, configuration.path, [datatype], force, indentation + 2, {})
plot_status = plotters_plot(webapi, configuration.path, [datatype], False, False,
data.as_dict()[the_data], output_name, None, indentation + 2, format_cache)
status += pl_status
status += plot_status
return status
def process(args):
config = args['config']
......@@ -486,6 +613,11 @@ def process(args):
return common.dot_diagram(config.path, 'experiment',
names, args['--path'], [])
elif args['plot']:
with common.make_webapi(config) as webapi:
return plot(webapi, config, 'experiment', names, args['--remote'],
force, 0, {}, args['--outputfolder'])
# Should not happen
logger.error("unrecognized `experiments' subcommand")
return 1
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.cmdline module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
"""Usage:
%(prog)s plotterparameters list
%(prog)s plotterparameters check [<name>]...
%(prog)s plotterparameters pull [--force] [<name>]...
%(prog)s plotterparameters create <name>...
%(prog)s plotterparameters version <name>
%(prog)s plotterparameters fork <src> <dst>
%(prog)s plotterparameters rm <name>...
%(prog)s plotterparameters --help
Commands:
list Lists all the plotterparameters available locally
check Checks a local plotterparameter for validity
create Creates a new local plotterparameter
version Creates a new version of an existing plotterparameter
fork Forks a local plotterparameter
rm Deletes a local plotterparameter
Options:
--force Performs operation regardless of conflicts
--dry-run Doesn't really perform the task, just comments what would do
--help Display this screen
"""
import simplejson
import collections
from .dataformats import pull as dataformats_pull
import logging
logger = logging.getLogger(__name__)
from . import common
def pull(webapi, prefix, names, force, indentation, format_cache):
"""Copies plotterparameters from the server.
Parameters:
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
prefix (str): A string representing the root of the path in which the
user objects are stored
names (list): A list of strings, each representing the unique relative
path of the objects to retrieve or a list of usernames from which to
retrieve objects. If the list is empty, then we pull all available
objects of a given type. If no user is set, then pull all public
objects of a given type.
force (bool): If set to ``True``, then overwrites local changes with the
remotely retrieved copies.
indentation (int): The indentation level, useful if this function is
called recursively while downloading different object types. This is
normally set to ``0`` (zero).
Returns:
int: Indicating the exit status of the command, to be reported back to
the calling process. This value should be zero if everything works OK,
otherwise, different than zero (POSIX compliance).
"""
# download required plotterparameter
status, names = common.pull(webapi, prefix, 'plotterparameter', names,
['data', 'short_description', 'plotter'], force,
indentation)
return status
def process(args):
if args['list']:
return common.display_local_list(args['config'].path, 'plotterparameter')
elif args['check']:
return common.check(args['config'].path, 'plotterparameter', args['<name>'])
elif args['pull']:
with common.make_webapi(args['config']) as webapi:
args['<name>'] = common.make_up_remote_list(webapi, 'plotterparameter',
args['<name>'])
if args['<name>'] is None: return 1 #error
return pull(webapi, args['config'].path, args['<name>'],
args['--force'], 0, {})
elif args['create']:
return common.create(args['config'].path, 'plotterparameter', args['<name>'])
elif args['version']:
return common.new_version(args['config'].path, 'plotterparameter',
args['<name>'][0])
elif args['fork']:
return common.fork(args['config'].path, 'plotterparameter', args['<src>'],
args['<dst>'])
elif args['rm']:
return common.delete_local(args['config'].path, 'plotterparameter', args['<name>'])
return 1
......@@ -29,6 +29,8 @@
"""Usage:
%(prog)s plotters list
%(prog)s plotters check [<name>]...
%(prog)s plotters pull [--force] [<name>]...
%(prog)s plotters plot [--force] [--sample_data] [--inputdata=<filename.json>] [--outputimage=<filename.png>] [--plotterparameter=<plotterparameter>] [<name>]...
%(prog)s plotters create <name>...
%(prog)s plotters version <name>
%(prog)s plotters fork <src> <dst>
......@@ -52,11 +54,214 @@ Options:
"""
import os
import simplejson
import collections
from .dataformats import pull as dataformats_pull
from .plotterparameters import pull as plotterparameters_pull
from .libraries import pull as libraries_pull
import logging
logger = logging.getLogger(__name__)
from . import common
from beat.core import plotter
from beat.core import dataformat
def pull(webapi, prefix, names, force, indentation, format_cache):
"""Copies plotters from the server.
Parameters:
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
prefix (str): A string representing the root of the path in which the
user objects are stored
names (:py:class:`list`): A list of strings, each representing the unique relative
path of the objects to retrieve or a list of usernames from which to
retrieve objects. If the list is empty, then we pull all available
objects of a given type. If no user is set, then pull all public
objects of a given type.
force (bool): If set to ``True``, then overwrites local changes with the
remotely retrieved copies.
indentation (int): The indentation level, useful if this function is
called recursively while downloading different object types. This is
normally set to ``0`` (zero).
Returns:
int: Indicating the exit status of the command, to be reported back to
the calling process. This value should be zero if everything works OK,
otherwise, different than zero (POSIX compliance).
"""
# download required plotter
status, names = common.pull(webapi, prefix, 'plotter', names,
['declaration', 'code'], force,
indentation)
# check dataformat, libraries, default plotterparameter used by plotter
dataformats = []
libraries = []
for name in names:
data = common.fetch_object(webapi, 'plotter', name, ['declaration', 'uses'])
data = simplejson.loads(data['declaration'], object_pairs_hook=collections.OrderedDict)
libraries_json = simplejson.loads(simplejson.dumps(data['uses']))
for key, val in libraries_json.items():
libraries.append(val)
lb_status = libraries_pull(webapi, prefix, libraries, force,
indentation + 2, format_cache)
dataformats.append(data['dataformat'])
# downloads any formats to which we depend on
df_status = dataformats_pull(webapi, prefix, dataformats, force,
indentation + 2, format_cache)
# download default plotterparameter
defaultplotters = common.fetch_object(webapi, 'defaultplotter', None, ['parameter'])
for df in defaultplotters:
if name == df['plotter']:
plp_status = plotterparameters_pull(webapi, prefix, [df['parameter']], ['data', 'short_description', 'plotter'], force, indentation + 2)
return status + lb_status + df_status
def plot(webapi, prefix, names, force, need_data_sample, inputdata, outputimage, plotterparameter, indentation, format_cache):
"""plot sample plot from the server.
Parameters:
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
prefix (str): A string representing the root of the path in which the
user objects are stored
names (:py:class:`list`): A list of strings, each representing the unique relative
path of the objects to retrieve or a list of usernames from which to
retrieve objects. If the list is empty, then we pull all available
objects of a given type. If no user is set, then pull all public
objects of a given type.
need_data_sample (bool): If set to ``True``, then fetch sample data
for plotter.
force (bool): If set to ``True``, then overwrites local changes with the
remotely retrieved copies.
inputdata (str): The path to the json file containing data to
be plotted.
outputimage (str): The png filename for dumping the plot. If not set
the default value is "output_image.png".
plotterparameter (str): The name of the plotterparameter (without ".json"
extension).
indentation (int): The indentation level, useful if this function is
called recursively while downloading different object types. This is
normally set to ``0`` (zero).
Returns:
int: Indicating the exit status of the command, to be reported back to
the calling process. This value should be zero if everything works OK,
otherwise, different than zero (POSIX compliance).
"""
indent = indentation * ' '
name = names[0]
# create a core_plotter
core_plotter = plotter.Plotter(prefix, name)
if not core_plotter.valid:
message = 'Plotter %s is invalid' % core_plotter.name
print(message)
import sys
sys.exit()
sample_data = None
# get sample data
if need_data_sample:
sample_data = common.fetch_object(webapi, 'plotter', name, ['sample_data'])
sample_data = simplejson.loads(sample_data['sample_data'])
# provide data
if inputdata is None and not need_data_sample:
message = 'Error: Missing --sample_data argument or inputdata'
print(message)
import sys
sys.exit()
elif inputdata is not None:
if type(inputdata) is str:
with open(os.path.join(prefix, inputdata), 'r') as f:
sample_data = simplejson.load(f)
f.closed
elif type(inputdata) is dict:
sample_data = inputdata
else:
message = 'Error: inputdata should be dict or str type'
print(message)
import sys
sys.exit()
# output
plotter_path = os.path.join(prefix, common.TYPE_PLURAL['plotter'], name.rsplit('/', 1)[0])
outputimage_name = "output_image.png"
if outputimage is not None:
outputimage_name = outputimage
else:
outputimage_name = os.path.join(plotter_path, outputimage_name)
# plot
corefmt = dataformat.DataFormat(prefix, core_plotter.dataformat.name)
sample_data = corefmt.type().from_dict(sample_data, casting='unsafe')
# plotterparameter
data = None
if plotterparameter is not None:
try:
with open(os.path.join(prefix, common.TYPE_NOSTORAGE['plotterparameter'], plotterparameter + '.json'), 'r') as f:
data = simplejson.load(f, object_pairs_hook=collections.OrderedDict)