Commit fee7ca1b authored by André Anjos's avatar André Anjos 💬

Merge branch '82_duplicate_key_handling' into 'master'

Duplicate key handling

Closes #82

See merge request !78
parents b97ea68a c595eb6d
Pipeline #31395 passed with stage
in 18 minutes and 12 seconds
...@@ -37,13 +37,14 @@ ...@@ -37,13 +37,14 @@
import os import os
import collections
import pkg_resources import pkg_resources
import six import six
import simplejson as json import simplejson as json
import jsonschema import jsonschema
from beat.backend.python.utils import error_on_duplicate_key_hook
def maybe_load_json(s): def maybe_load_json(s):
"""Maybe loads the JSON from a string or filename""" """Maybe loads the JSON from a string or filename"""
...@@ -55,7 +56,7 @@ def maybe_load_json(s): ...@@ -55,7 +56,7 @@ def maybe_load_json(s):
with open(s, "rt") as f: with open(s, "rt") as f:
return maybe_load_json(f) return maybe_load_json(f)
else: else:
return json.loads(s, object_pairs_hook=collections.OrderedDict) return json.loads(s, object_pairs_hook=error_on_duplicate_key_hook)
# if it is a 'file-like' object # if it is a 'file-like' object
if hasattr(s, "read"): if hasattr(s, "read"):
...@@ -163,6 +164,8 @@ def validate(schema_name, data): ...@@ -163,6 +164,8 @@ def validate(schema_name, data):
data = maybe_load_json(data) data = maybe_load_json(data)
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
return data, ["invalid JSON code: %s" % str(e)] return data, ["invalid JSON code: %s" % str(e)]
except RuntimeError as e:
return data, ["Invalid JSON: %s" % str(e)]
# handles the schema version # handles the schema version
if schema_name != "dataformat": if schema_name != "dataformat":
......
{
"schema_version": 3,
"language": "python",
"api_version": 2,
"type": "loop",
"type": "loop_user",
"groups": [
{
"inputs": {
"in": {
"type": "user/single_integer/1"
}
},
"loop": {
"answer": {
"type": "user/single_integer/1"
}
}
}
]
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
class Algorithm:
def validate(self, result):
value = result.value
return value < 6
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
"language": "python", "language": "python",
"api_version": 2, "api_version": 2,
"type": "loop", "type": "loop",
"splittable": false,
"groups": [ "groups": [
{ {
"inputs": { "inputs": {
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
], ],
"parameters": { "parameters": {
"threshold": { "threshold": {
"default": "9", "default": 9,
"type": "int8", "type": "int8",
"description": "Value that will change loop result" "description": "Value that will change loop result"
} }
......
...@@ -4,9 +4,6 @@ ...@@ -4,9 +4,6 @@
"api_version": 2, "api_version": 2,
"type": "loop_user", "type": "loop_user",
"splittable": false, "splittable": false,
"parameters": {
},
"groups": [ "groups": [
{ {
"inputs": { "inputs": {
......
...@@ -17,6 +17,5 @@ ...@@ -17,6 +17,5 @@
} }
} }
} }
], ]
"uses": {}
} }
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
], ],
"parameters": { "parameters": {
"offset": { "offset": {
"default": "1", "default": 1,
"type": "int32" "type": "int32"
} }
}, },
......
...@@ -18,7 +18,5 @@ ...@@ -18,7 +18,5 @@
} }
} }
} }
], ]
"parameters": {
}
} }
...@@ -11,7 +11,8 @@ ...@@ -11,7 +11,8 @@
], ],
"parameters": { "parameters": {
"command": { "command": {
"type": "string" "type": "string",
"default": ""
} }
}, },
"results": { "results": {
......
{ {
"globals": {
},
"blocks": { "blocks": {
"addition": { "addition": {
"algorithm": "user/sum/1", "algorithm": "user/sum/1",
......
{
"language": "python",
"language": "cxx"
}
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###################################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# Redistribution and use in source and binary forms, with or without #
# modification, are permitted provided that the following conditions are met: #
# #
# 1. Redistributions of source code must retain the above copyright notice, this #
# list of conditions and the following disclaimer. #
# #
# 2. Redistributions in binary form must reproduce the above copyright notice, #
# this list of conditions and the following disclaimer in the documentation #
# and/or other materials provided with the distribution. #
# #
# 3. Neither the name of the copyright holder nor the names of its contributors #
# may be used to endorse or promote products derived from this software without #
# specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
# #
###################################################################################
import sys
def f():
return "OK"
def pyver():
return "%d.%d" % sys.version_info[:2]
...@@ -68,14 +68,14 @@ def test_dependencies(): ...@@ -68,14 +68,14 @@ def test_dependencies():
alg = Algorithm(tmp_prefix, name) alg = Algorithm(tmp_prefix, name)
nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors)) nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors))
nose.tools.eq_(len(alg.uses), 0) nose.tools.assert_is_none(alg.uses)
nose.tools.eq_(len(alg.libraries), 0) nose.tools.eq_(len(alg.libraries), 0)
l_dep = Library(tmp_prefix, dep_name) l_dep = Library(tmp_prefix, dep_name)
nose.tools.assert_true(l_dep.valid, "\n * %s" % "\n * ".join(l_dep.errors)) nose.tools.assert_true(l_dep.valid, "\n * %s" % "\n * ".join(l_dep.errors))
# check modification # check modification
alg.uses["dep1"] = dep_name alg.uses = {"dep1": dep_name}
alg.write() alg.write()
alg = Algorithm(tmp_prefix, name) alg = Algorithm(tmp_prefix, name)
nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors)) nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors))
...@@ -115,14 +115,28 @@ def test_invalid_dependencies(): ...@@ -115,14 +115,28 @@ def test_invalid_dependencies():
alg = Algorithm(tmp_prefix, name) alg = Algorithm(tmp_prefix, name)
nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors)) nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors))
nose.tools.eq_(len(alg.uses), 0) nose.tools.assert_is_none(alg.uses)
nose.tools.eq_(len(alg.libraries), 0) nose.tools.eq_(len(alg.libraries), 0)
l_dep = Library(tmp_prefix, "errors/invalid_dep/1") l_dep = Library(tmp_prefix, "errors/invalid_dep/1")
nose.tools.assert_true(l_dep.valid, "\n * %s" % "\n * ".join(l_dep.errors)) nose.tools.assert_true(l_dep.valid, "\n * %s" % "\n * ".join(l_dep.errors))
alg.uses["dep"] = dep_name alg.uses = {"dep": dep_name}
alg.write() alg.write()
alg = Algorithm(tmp_prefix, name) alg = Algorithm(tmp_prefix, name)
nose.tools.assert_false(alg.valid) nose.tools.assert_false(alg.valid)
nose.tools.assert_not_equal(alg.errors[0].find("differs from current language"), -1) nose.tools.assert_not_equal(alg.errors[0].find("differs from current language"), -1)
@nose.tools.with_setup(teardown=cleanup)
def test_invalid_dependency_setup():
name = "user/for_dep/1"
dep_name = "user/dep/1"
copy_objects(name, dep_name)
alg = Algorithm(tmp_prefix, name)
nose.tools.assert_true(alg.valid, "\n * %s" % "\n * ".join(alg.errors))
nose.tools.assert_is_none(alg.uses)
with nose.tools.assert_raises(RuntimeError):
alg.uses = "dummy"
...@@ -92,6 +92,16 @@ def test_invalid_loop_channel(): ...@@ -92,6 +92,16 @@ def test_invalid_loop_channel():
# ---------------------------------------------------------- # ----------------------------------------------------------
def test_duplicate_key():
algorithm = Algorithm(prefix, "schema/invalid_duplicate_key/1")
nose.tools.assert_false(algorithm.valid)
nose.tools.assert_not_equal(algorithm.errors[0].find("Invalid file content"), -1)
# ----------------------------------------------------------
def test_v2(): def test_v2():
algorithm = Algorithm(prefix, "user/integers_add_v2/1") algorithm = Algorithm(prefix, "user/integers_add_v2/1")
......
...@@ -115,6 +115,14 @@ def test_invalid_mix(): ...@@ -115,6 +115,14 @@ def test_invalid_mix():
nose.tools.assert_not_equal(lib.errors[0].find("differs from current language"), -1) nose.tools.assert_not_equal(lib.errors[0].find("differs from current language"), -1)
def test_duplicate_key():
lib = Library(prefix, "errors/duplicate_key/1")
nose.tools.assert_false(lib.valid)
nose.tools.eq_(len(lib.errors), 1)
nose.tools.assert_not_equal(lib.errors[0].find("found several times"), -1)
@nose.tools.with_setup(teardown=cleanup) @nose.tools.with_setup(teardown=cleanup)
def test_dependencies(): def test_dependencies():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment