Commit 3c0fe03d authored by André Anjos's avatar André Anjos 💬

Merge branch '25_handle_duplicate_key' into 'master'

Handle duplicate key

Closes #25

See merge request !51
parents 67ef00a1 176cf6fb
Pipeline #31394 passed with stage
in 22 minutes and 13 seconds
......@@ -48,7 +48,7 @@ import logging
import six
import numpy
import simplejson
import simplejson as json
from . import dataformat
from . import library
......@@ -433,7 +433,14 @@ class Algorithm(object):
return
with open(json_path, "rb") as f:
self.data = simplejson.loads(f.read().decode("utf-8"))
try:
self.data = json.loads(
f.read().decode("utf-8"),
object_pairs_hook=utils.error_on_duplicate_key_hook,
)
except RuntimeError as error:
self.errors.append("Algorithm declaration file invalid: %s" % error)
return
self.code_path = self.storage.code.path
self.code = self.storage.code.load()
......@@ -753,6 +760,8 @@ class Algorithm(object):
@uses.setter
def uses(self, value):
if not isinstance(value, dict):
raise RuntimeError("Invalid uses entry, must be a dict")
self.data["uses"] = value
return value
......@@ -936,7 +945,7 @@ class Algorithm(object):
"""
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
......
......@@ -46,7 +46,7 @@ import os
import sys
import six
import simplejson
import simplejson as json
import itertools
import numpy as np
from collections import namedtuple
......@@ -153,7 +153,7 @@ class Runner(object):
os.makedirs(os.path.dirname(filename))
with open(filename, "wb") as f:
data = simplejson.dumps(objs, cls=utils.NumpyJSONEncoder)
data = json.dumps(objs, cls=utils.NumpyJSONEncoder)
f.write(data.encode("utf-8"))
def setup(self, filename, start_index=None, end_index=None, pack=True):
......@@ -163,7 +163,10 @@ class Runner(object):
return
with open(filename, "rb") as f:
objs = simplejson.loads(f.read().decode("utf-8"))
objs = json.loads(
f.read().decode("utf-8"),
object_pairs_hook=utils.error_on_duplicate_key_hook,
)
Entry = namedtuple("Entry", sorted(objs[0].keys()))
objs = [Entry(**x) for x in objs]
......@@ -303,7 +306,14 @@ class Database(object):
return
with open(json_path, "rb") as f:
self.data = simplejson.loads(f.read().decode("utf-8"))
try:
self.data = json.loads(
f.read().decode("utf-8"),
object_pairs_hook=utils.error_on_duplicate_key_hook,
)
except RuntimeError as error:
self.errors.append("Database declaration file invalid: %s" % error)
return
self.code_path = self.storage.code.path
self.code = self.storage.code.load()
......@@ -542,7 +552,7 @@ class Database(object):
"""
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
......
......@@ -47,7 +47,7 @@ import copy
import six
import numpy
import simplejson
import simplejson as json
from . import utils
from .baseformat import baseformat
......@@ -186,7 +186,16 @@ class DataFormat(object):
return
with open(json_path, "rb") as f:
self.data = simplejson.loads(f.read().decode("utf-8"))
try:
self.data = json.loads(
f.read().decode("utf-8"),
object_pairs_hook=utils.error_on_duplicate_key_hook,
)
except RuntimeError as error:
self.errors.append(
"Dataformat declaration file invalid: %s" % error
)
return
dataformat_cache[self._name] = self # registers itself into the cache
......@@ -457,7 +466,7 @@ class DataFormat(object):
"""
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
......
......@@ -43,7 +43,7 @@ Validation for libraries
"""
import os
import simplejson
import simplejson as json
from . import loader
from . import utils
......@@ -164,7 +164,14 @@ class Library(object):
return
with open(json_path, "rb") as f:
self.data = simplejson.loads(f.read().decode("utf-8"))
try:
self.data = json.loads(
f.read().decode("utf-8"),
object_pairs_hook=utils.error_on_duplicate_key_hook,
)
except RuntimeError as error:
self.errors.append("Library declaration file invalid: %s" % error)
return
self.code_path = self.storage.code.path
......@@ -318,7 +325,7 @@ class Library(object):
"""
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
......
......@@ -42,7 +42,7 @@ protocoltemplates
Validation of database protocol templates
"""
import simplejson
import simplejson as json
from .dataformat import DataFormat
......@@ -142,7 +142,15 @@ class ProtocolTemplate(object):
return
with open(json_path, "rt") as f:
self.data = simplejson.loads(f.read())
try:
self.data = json.loads(
f.read(), object_pairs_hook=utils.error_on_duplicate_key_hook
)
except RuntimeError as error:
self.errors.append(
"Protocol template declaration file invalid: %s" % error
)
return
for set_ in self.data["sets"]:
......@@ -238,7 +246,7 @@ class ProtocolTemplate(object):
"""
return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
def __str__(self):
return self.json_dumps()
......
{
"schema_version": 2,
"schema_version": 3,
"language": "python",
"api_version": 2,
"type": "loop",
......
......@@ -4,9 +4,6 @@
"api_version": 2,
"type": "loop_user",
"splittable": false,
"parameters": {
},
"groups": [
{
"inputs": {
......
{
"language": "python",
"splittable": false,
"splittable": true,
"groups": [
{
"inputs": {
"in": {
"type": "user/single_integer/1"
}
},
"outputs": {
"out": {
"type": "user/single_integer/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
class Algorithm:
def process(self, inputs, outputs):
return True
......@@ -224,7 +224,7 @@
"default": 0.0,
"type": "float32",
"range": [
"-10.0",
-10.0,
10.0
]
},
......@@ -232,8 +232,8 @@
"default": 0.0,
"type": "float32",
"choice": [
"-10.0",
"-5.0",
-10.0,
-5.0,
0.0,
5.0,
10.0
......@@ -247,7 +247,7 @@
"default": 0.0,
"type": "float64",
"range": [
"-100.0",
-100.0,
100.0
]
},
......
{
"root_folder": "/tmp/foo/bar",
"protocols": [
{
"name": "test_duplicate_key",
"name": "test_duplicate_key",
"template": "double/1",
"views": {
"double": {
"view": "MyView",
"parameters": {
"threshold": 3
}
}
}
}
],
"schema_version": 2
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import numpy
from collections import namedtuple
from beat.backend.python.database import View as BaseView
class MyView(BaseView):
def index(self, root_folder, parameters):
Entry = namedtuple("Entry", ["out"])
return [Entry(42)]
def get(self, output, index):
obj = self.objs[index]
if output == "out":
return {"value": numpy.int32(obj.out)}
{
"schema_version": 1,
"sets": [
{
"name": "labelled",
"name": "labelled1",
"outputs": {
"value": "user/single_integer/1",
"label": "user/single_string/1"
}
}
]
}
......@@ -92,6 +92,13 @@ class TestLegacyAPI_Loading(unittest.TestCase):
with self.assertRaises(AttributeError):
algorithm.runner()
def test_duplicate_key_error(self):
algorithm = Algorithm(prefix, "legacy/duplicate_key_error/1")
self.assertFalse(algorithm.valid)
self.assertNotEqual(
algorithm.errors[0].find("Algorithm declaration file invalid"), -1
)
def test_load_valid_algorithm(self):
algorithm = Algorithm(prefix, "legacy/valid_algorithm/1")
self.assertEqual(algorithm.name, "legacy/valid_algorithm/1")
......
......@@ -149,4 +149,15 @@ def compare_definitions(db_name, protocol_name, view_name):
db_2 = load("{}/2".format(db_name))
db_2_view_definition = db_2.view_definition(protocol_name, view_name)
nose.tools.eq_(db_1_view_definition, db_2_view_definition)
db_1_sorted = sorted(db_1_view_definition)
db_2_sorted = sorted(db_2_view_definition)
nose.tools.eq_(db_1_sorted, db_2_sorted)
# ----------------------------------------------------------
def test_duplicate_key_error():
database = Database(prefix, "duplicate_key_error/1")
nose.tools.assert_false(database.valid)
nose.tools.assert_true("Database declaration file invalid" in database.errors[0])
......@@ -346,3 +346,12 @@ def test_string():
copy.isclose(obj),
"%r is not close enough to %r" % (copy.as_dict(), obj.as_dict()),
)
# ----------------------------------------------------------
def test_duplicate_key_error():
df = DataFormat(prefix, "errors/duplicate_key/1")
nose.tools.assert_false(df.valid)
nose.tools.assert_true("Dataformat declaration file invalid" in df.errors[0])
......@@ -111,3 +111,14 @@ def test_load_protocol_with_two_sets():
nose.tools.assert_is_not_none(set_["outputs"]["b"])
nose.tools.assert_is_not_none(set_["outputs"]["c"])
nose.tools.assert_is_not_none(set_["outputs"]["sum"])
# ----------------------------------------------------------
def test_duplicate_key_error():
protocoltemplate = ProtocolTemplate(prefix, "duplicate_key_error/1")
nose.tools.assert_false(protocoltemplate.valid)
nose.tools.assert_true(
"Protocol template declaration file invalid" in protocoltemplate.errors[0]
)
......@@ -401,6 +401,26 @@ class NumpyJSONEncoder(simplejson.JSONEncoder):
# ----------------------------------------------------------
def error_on_duplicate_key_hook(pairs):
"""JSON loader hook that will error out if several same keys are found
Returns an OrderedDict if everything goes well
"""
dct = collections.OrderedDict()
for key, value in pairs:
if key in dct:
raise RuntimeError(
"Invalid file content\n{} found several times".format(key)
)
dct[key] = value
return dct
# ----------------------------------------------------------
def has_argument(method, argument):
try:
from inspect import signature
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment