Merge branch 'algorithm_cxx_v2' into 'master'

Algorithm cxx v2 (replaces !37)

See merge request !40

.gitlab-ci.yml

@@ -129,7 +129,7 @@ deploy_beta:
   <<: *deploy_job
   environment: beta
   only:
-    - master 
+    - master
 
 
 deploy_stable:

MANIFEST.in

@@ -4,4 +4,5 @@ recursive-include scripts *.sh
 recursive-include doc conf.py *.rst *.png *.svg *.ico *.odg *.pdf *.dot
 recursive-include beat/core/schema *.json
 recursive-include beat/core/prototypes *.json *.py
-recursive-include beat/core/test/prefix *.json *.py *.r *.m *.rst *.h *.cpp
+recursive-include beat/core/test/prefix *.json *.py *.r *.m *.rst *.h *.cpp CMakeLists.txt
+recursive-include beat/core/test/scripts *.sh

beat/core/dock.py

@@ -252,7 +252,6 @@ class Host(object):
         environments = {}
         db_environments = {}
 
-
         cmd = [
             'docker',
             'images',
@@ -347,6 +346,14 @@ class Host(object):
         if name:
             cmd.append(name)
 
+        workdir = container.workdir
+        if workdir:
+            cmd.append(workdir)
+
+        entrypoint = container.entrypoint
+        if entrypoint:
+            cmd.append(entrypoint)
+
         if container.image in Host.images_cache:
             image_infos = Host.images_cache[container.image]
             if ('capabilities' in image_infos) and ('gpu' in image_infos['capabilities']):
@@ -393,10 +400,12 @@ class Host(object):
         # Mount the volumes
         cmd.extend(container.volumes)
 
-
         # Expose the ports
         cmd.extend(container.ports)
 
+        # Environment variables
+        cmd.extend(container.environment_variables)
+
         cmd.append(container.image)
         cmd.extend(container.command)
 
@@ -624,8 +633,11 @@ class Container:
         self.id = None
         self._volumes = {}
         self._ports = {}
+        self._environment_variables = {}
         self._stats = None
         self._name = None
+        self._workdir = None
+        self._entrypoint = None
 
 
     def set_name(self, name):
@@ -635,6 +647,18 @@ class Container:
         self._name = name
 
 
+    def set_workdir(self, workdir):
+        """ Set the work folder to be used by the container
+        """
+        self._workdir = workdir
+
+
+    def set_entrypoint(self, entrypoint):
+        """ Set the entry point to be used by the container
+        """
+        self._entrypoint = entrypoint
+
+
     def add_volume(self, path, mount_path, read_only=True):
         """Add a volume to be mounted on the container
 
@@ -670,6 +694,17 @@ class Container:
 
         self._ports[container_port] = value
 
+    def add_environment_variable(self, name, value):
+        """Add an environment variable
+
+        Parameters:
+            :param str name: Name of the variable
+
+            :param str value: Content of the variable
+        """
+
+        self._environment_variables[name] = value
+
 
     def reset_ports(self):
         """Empty the port bindings"""
@@ -685,6 +720,22 @@ class Container:
         return name
 
 
+    @property
+    def workdir(self):
+        workdir = ''
+        if self._workdir:
+            workdir = '--workdir=%s' % self._workdir
+        return workdir
+
+
+    @property
+    def entrypoint(self):
+        entrypoint = ''
+        if self._entrypoint:
+            entrypoint = '--entrypoint=%s' % self._entrypoint
+        return entrypoint
+
+
     @property
     def volumes(self):
         """Returns the volumes of this container in a suitable form to build
@@ -724,6 +775,17 @@ class Container:
         return ports
 
 
+    @property
+    def environment_variables(self):
+        """Returns the environment variables to set on this container.
+        """
+
+        environment_variables = []
+        for k, v in self._environment_variables.items():
+            environment_variables.append('--env={}={}'.format(k, v))
+        return environment_variables
+
+
     @property
     def network(self):
         network = ''
@@ -753,7 +815,10 @@ class Container:
         cmd += "%s " % self.user
         cmd += ' '.join(self.volumes)
         cmd += ' '.join(self.ports)
+        cmd += ' '.join(self.environment_variables)
         cmd += "%s " % self.name
+        cmd += "%s " % self.workdir
+        cmd += "%s " % self.entrypoint
 
         cmd += "%s " % self.image
 

beat/core/experiment.py

@@ -380,8 +380,8 @@ class Experiment(object):
 
                 self.algorithms[algoname] = thisalgo
                 if thisalgo.errors:
-                    self.errors.append("/analyzers/%s: algorithm `%s' is invalid" % \
-                            (analyzername, algoname))
+                    self.errors.append("/analyzers/%s: algorithm `%s' is invalid:\n%s" % \
+                            (analyzername, algoname, "\n".join(thisalgo.errors)))
                     continue
 
             else:

beat/core/test/__init__.py

@@ -57,7 +57,9 @@ network_name = os.environ.get('DOCKER_TEST_NETWORK', 'beat_core_test_network')
 network = None
 
 # Setup the logging system
-if False:
+VERBOSE_TEST_LOGGING = os.environ.get('VERBOSE_TEST_LOGGING', False) == 'True'
+
+if VERBOSE_TEST_LOGGING:
     formatter = logging.Formatter(fmt="[%(asctime)s - TESTS - "
                                       "%(name)s] %(levelname)s: %(message)s",
                                   datefmt="%d/%b/%Y %H:%M:%S")

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/1.json

+{
+  "language": "cxx",
+  "groups": [
+    {
+      "name": "main",
+      "inputs": {
+        "in_data": {
+          "type": "user/single_integer/1"
+        }
+      }
+    }
+  ],
+  "results": {
+    "out_data": {
+      "type": "int32",
+      "display": false
+    }
+  }
+}

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+project(BEAT_CORE_CXX_INTEGERS_ECHO_ANALYZER)
+
+
+set(BEAT_BACKEND_CXX_DIR "/usr/local/beat")
+
+
+# CMake setup
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+if (COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+elseif (COMPILER_SUPPORTS_CXX0X)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+else()
+    message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
+endif()
+
+
+# Retrieve the dependencies
+find_package(Boost REQUIRED)
+
+
+# Setup the search paths
+include_directories(
+    "${BEAT_BACKEND_CXX_DIR}/include"
+    "${Boost_INCLUDE_DIRS}"
+)
+
+link_directories(
+    "${BEAT_BACKEND_CXX_DIR}/bin"
+)
+
+
+# List the source files
+set(HEADERS "algorithm.h"
+            "beat_setup.h"
+            "user_single_integer_1.h"
+)
+
+set(SRCS "algorithm.cpp"
+         "beat_setup.cpp"
+         "user_single_integer_1.cpp"
+)
+
+# Create and link the library
+add_library(cxx_integers_echo_analyzer SHARED ${SRCS} ${HEADERS})
+
+target_link_libraries(cxx_integers_echo_analyzer beat_backend_cxx)
+set_target_properties(cxx_integers_echo_analyzer PROPERTIES
+                                                 COMPILE_FLAGS "-fPIC"
+                                                 OUTPUT_NAME "1"
+                                                 PREFIX ""
+                                                 LIBRARY_OUTPUT_DIRECTORY "${BEAT_CORE_CXX_INTEGERS_ECHO_ANALYZER_SOURCE_DIR}"
+)

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/algorithm.cpp

+// NOTE: This file implements the algorithm declared in the file
+// 'user/cxx_integers_echo/1.json'
+
+#include "algorithm.h"
+#include "beat.backend.cxx/input.h"
+#include "beat.backend.cxx/inputlist.h"
+#include "beat.backend.cxx/outputlist.h"
+#include "beat.backend.cxx/dataloader.h"
+#include "user_single_integer_1.h"
+
+using namespace beat::backend::cxx;
+
+
+Algorithm::Algorithm()
+{
+}
+
+//---------------------------------------------------------
+
+Algorithm::~Algorithm()
+{
+}
+
+//---------------------------------------------------------
+
+bool Algorithm::setup(const json& parameters)
+{
+    return true;
+}
+
+//---------------------------------------------------------
+
+bool Algorithm::prepare(const DataLoaderList& data_load_list)
+{
+    return true;
+}
+
+//---------------------------------------------------------
+
+bool Algorithm::process(const InputList& inputs,
+                        const DataLoaderList& data_load_list,
+                        const OutputList& outputs)
+{
+    auto data = inputs["in_data"]->data<dataformats::user::single_integer_1>();
+
+    outputs["out_data"]->write(data);
+
+    return true;
+}
+
+//---------------------------------------------------------
+
+IAlgorithm* create_algorithm()
+{
+    return new Algorithm();
+}

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/algorithm.h

+// NOTE: This file implements the algorithm declared in the file
+// 'user/cxx_integers_echo/1.json'
+
+#ifndef _BEAT_GENERATED_ALGORITHM_H_
+#define _BEAT_GENERATED_ALGORITHM_H_
+
+#include <beat.backend.cxx/algorithm.h>
+
+
+class Algorithm: public beat::backend::cxx::IAlgorithmSequential
+{
+public:
+    Algorithm();
+    virtual ~Algorithm();
+
+    bool setup(const json& parameters) override;
+
+    bool prepare(const beat::backend::cxx::DataLoaderList& data_load_list) override;
+	bool process(const beat::backend::cxx::InputList& inputs,
+                 const beat::backend::cxx::DataLoaderList& data_load_list,
+                 const beat::backend::cxx::OutputList& outputs) override;
+
+};
+
+
+extern "C" {
+    beat::backend::cxx::IAlgorithm* create_algorithm();
+}
+
+#endif

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/beat_setup.cpp

+// NOTE: This file was automatically generated from the algorithm declaration file
+// 'user/cxx_integers_echo/1.json'
+
+#include <beat.backend.cxx/dataformatfactory.h>
+#include "beat_setup.h"
+#include "user_single_integer_1.h"
+
+using namespace beat::backend::cxx;
+
+
+void setup_beat_cxx_module()
+{
+    DataFormatFactory* factory = DataFormatFactory::getInstance();
+
+    factory->registerDataFormat<dataformats::user::single_integer_1>();
+}

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/beat_setup.h

+// NOTE: This file was automatically generated from the algorithm declaration file
+// 'user/cxx_integers_echo/1.json'
+
+#ifndef _BEAT_SETUP_H_
+#define _BEAT_SETUP_H_
+
+extern "C" {
+    void setup_beat_cxx_module();
+}
+
+#endif

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/user_single_integer_1.cpp

+// NOTE: This file was automatically generated from the dataformat declaration file
+// 'user/single_integer/1.json'
+
+#include "user_single_integer_1.h"
+#include <beat.backend.cxx/dataformatfactory.h>
+
+using namespace beat::backend::cxx;
+
+dataformats::user::single_integer_1::single_integer_1()
+{
+}
+
+//---------------------------------------------------------
+
+size_t dataformats::user::single_integer_1::fixed_size()
+{
+    return sizeof(int32_t);
+}
+
+//---------------------------------------------------------
+
+size_t dataformats::user::single_integer_1::size() const
+{
+    return dataformats::user::single_integer_1::fixed_size();
+}
+
+//---------------------------------------------------------
+
+void dataformats::user::single_integer_1::pack(uint8_t** buffer) const
+{
+    beat::backend::cxx::pack(value, buffer);
+}
+
+//---------------------------------------------------------
+
+void dataformats::user::single_integer_1::unpack(uint8_t** buffer)
+{
+    value = beat::backend::cxx::unpack_scalar<int32_t>(buffer);
+}
+
+//---------------------------------------------------------
+
+Data* dataformats::user::single_integer_1::create()
+{
+    return new user::single_integer_1();
+}
+
+//---------------------------------------------------------
+
+const char *dataformats::user::single_integer_1::getNameStatic()
+{
+    return "user/single_integer/1";
+}
+
+//---------------------------------------------------------
+
+std::string dataformats::user::single_integer_1::toJson() const
+{
+return "{"
+"\"field\":\"int32_t\""
+"}";
+}

beat/core/test/prefix/algorithms/user/cxx_integers_echo_analyzer/user_single_integer_1.h

+// NOTE: This file was automatically generated from the dataformat declaration file
+// 'user/single_integer/1.json'
+
+#ifndef _BEAT_GENERATED_user_single_integer_1_H_
+#define _BEAT_GENERATED_user_single_integer_1_H_
+
+#include <beat.backend.cxx/data.h>
+
+namespace dataformats {
+namespace user {
+
+class single_integer_1: public beat::backend::cxx::DataImpl<single_integer_1>
+{
+public:
+    single_integer_1();
+
+	static size_t fixed_size();
+
+    size_t size() const override;
+    void pack(uint8_t** buffer) const override;
+    void unpack(uint8_t** buffer) override;
+
+    static Data* create();
+    static const char *getNameStatic();
+
+    std::string toJson() const override;
+
+public:
+    int32_t value;
+};
+
+} // user
+} // dataformats
+
+#endif

beat/core/test/prefix/algorithms/user/cxx_integers_echo_autonomous/1.json

+{
+  "schema_version": 2,
+  "language": "cxx",
+  "api_version": 2,
+  "type": "autonomous",
+  "splittable": true,
+  "groups": [
+    {
+      "name": "main",
+      "inputs": {
+        "in_data": {
+          "type": "user/single_integer/1"
+        }
+      },
+      "outputs": {
+        "out_data": {
+          "type": "user/single_integer/1"
+        }
+      }
+    }
+  ]
+}

beat/core/test/prefix/algorithms/user/cxx_integers_echo_autonomous/CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+project(BEAT_CORE_CXX_INTEGERS_ECHO_AUTONOMOUS)
+
+
+set(BEAT_BACKEND_CXX_DIR "/usr/local/beat")
+
+
+# CMake setup
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+if (COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+elseif (COMPILER_SUPPORTS_CXX0X)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+else()
+    message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
+endif()
+
+
+# Retrieve the dependencies
+find_package(Boost REQUIRED)
+
+
+# Setup the search paths
+include_directories(
+    "${BEAT_BACKEND_CXX_DIR}/include"
+    "${Boost_INCLUDE_DIRS}"
+)
+
+link_directories(
+    "${BEAT_BACKEND_CXX_DIR}/bin"
+)
+
+
+# List the source files
+set(HEADERS "algorithm.h"
+            "beat_setup.h"
+            "user_single_integer_1.h"
+)
+
+set(SRCS "algorithm.cpp"
+         "beat_setup.cpp"
+         "user_single_integer_1.cpp"
+)
+
+# Create and link the library
+add_library(cxx_integers_echo_autonomous SHARED ${SRCS} ${HEADERS})
+
+target_link_libraries(cxx_integers_echo_autonomous beat_backend_cxx)
+set_target_properties(cxx_integers_echo_autonomous PROPERTIES
+                                                   COMPILE_FLAGS "-fPIC"
+                                                   OUTPUT_NAME "1"
+                                                   PREFIX ""
+                                                   LIBRARY_OUTPUT_DIRECTORY "${BEAT_CORE_CXX_INTEGERS_ECHO_AUTONOMOUS_SOURCE_DIR}"
+)

beat/core/test/prefix/algorithms/user/cxx_integers_echo_autonomous/algorithm.cpp

+// NOTE: This file implements the algorithm declared in the file
+// 'user/cxx_integers_echo/1.json'
+
+#include "algorithm.h"
+#include "beat.backend.cxx/outputlist.h"
+#include "beat.backend.cxx/dataloader.h"
+#include "user_single_integer_1.h"
+#include <iostream>
+
+using namespace beat::backend::cxx;
+
+
+Algorithm::Algorithm()
+{
+}
+
+//---------------------------------------------------------
+
+Algorithm::~Algorithm()
+{
+}
+
+//---------------------------------------------------------
+
+bool Algorithm::setup(const json& parameters)
+{
+    return true;
+}
+
+//---------------------------------------------------------
+
+bool Algorithm::prepare(const DataLoaderList& data_load_list)
+{
+    return true;
+}
+
+//---------------------------------------------------------
+
+bool Algorithm::process(const DataLoaderList& data_load_list, const OutputList& outputs)
+{
+
+    auto data_loader = data_load_list.loader_of("in_data");