[widgets][experimenteditor] removed prefix modelization for algorithms/experimenteditor

beat/editor/backend/resourcemodels.py

@@ -40,10 +40,13 @@ EDITED_KEY = "edited"
 
 
 # ------------------------------------------------------------------------------
+
 # Prefix modelization
 
 
 class ExperimentResources:
+    """Modelization of the experiments resources"""
+
     def __init__(self, context=None):
         self.context = context
 
@@ -65,20 +68,27 @@ class ExperimentResources:
         self.refresh()
 
     def refresh(self):
-        if self.context is None:
-            return
-
         ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
         ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
         INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
+        QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
+        QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
+        INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
 
         query = QSqlQuery()
 
-        if not query.exec_(ALGORITHM_TABLE_CLEANUP):
+        for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
+            if not query.exec_(query_str):
                 raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
 
-        if not query.exec_(ALGORITHM_TABLE):
-            raise RuntimeError(f"Failed to create table: {query.lastError().text()}")
+        for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
+            if not query.exec_(query_str):
+                raise RuntimeError(
+                    f"Failed to create table: {query.lastError().text()}"
+                )
+
+        if self.context is None:
+            return
 
         prefix_path = self.context.meta["config"].path
         model = AssetModel()
@@ -113,6 +123,40 @@ class ExperimentResources:
                     f"Failed to insert algorithm: {query.lastError().text()}"
                 )
 
+        if not query.prepare(INSERT_QUEUE):
+            raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
+
+        environments_path = self.context.meta["environments"]
+        with open(environments_path, "rt") as file:
+            environment_data = json.load(file)
+
+        for item in environment_data.get("remote", []):
+            env_name = item["name"]
+            env_version = item["version"]
+            # import ipdb; ipdb.set_trace()
+            for name in item["queues"].keys():
+                query.addBindValue(name)
+                query.addBindValue(env_name)
+                query.addBindValue(env_version)
+                query.addBindValue("remote")
+
+                if not query.exec_():
+                    raise RuntimeError(
+                        f"Failed to insert queue: {query.lastError().text()}"
+                    )
+
+        for _, image_info in environment_data.get("docker", {}).items():
+            env_name = image_info["name"]
+            env_version = image_info["version"]
+            query.addBindValue("Local")
+            query.addBindValue(env_name)
+            query.addBindValue(env_version)
+            query.addBindValue("docker")
+            if not query.exec_():
+                raise RuntimeError(
+                    f"Failed to insert queue: {query.lastError().text()}"
+                )
+
 
 class AlgorithmResourceModel(QSqlTableModel):
     def __init__(self, parent=None):
@@ -168,3 +212,55 @@ class AlgorithmResourceModel(QSqlTableModel):
 
         self._types = type_list
         self.update()
+
+
+class QueueResourceModel(QSqlTableModel):
+    def __init__(self, parent=None):
+        super().__init__(parent=parent)
+
+        self._environment = None
+        self._version = None
+        self._type = None
+
+        self.setTable("queues")
+        self.select()
+        self.update()
+
+    def update(self):
+        filter_str = ""
+        if self._environment is not None:
+            filter_str += f"env_name='{self._environment}'"
+
+        if self._version is not None:
+            if filter_str:
+                filter_str += " AND "
+            filter_str += f"env_version='{self._version}'"
+
+        if self._type is not None:
+            if filter_str:
+                filter_str += " AND "
+            filter_str += f"env_type='{self._type}'"
+
+        self.setFilter(filter_str)
+
+    def setEnvironment(self, name, version):
+        if self._environment == name and self._version == version:
+            return
+
+        self._environment = name
+        self._version = version
+        self.update()
+
+    def setType(self, type_):
+        if self._type == type_:
+            return
+
+        self._type = type_
+        self.update()
+
+    def dump(self):
+        print(self.filter())
+        for i in range(self.rowCount()):
+            print([self.index(i, j).data() for j in range(4)])
+
+

beat/editor/widgets/experimenteditor.py

@@ -52,10 +52,6 @@ from PyQt5.QtWidgets import QTabWidget
 from PyQt5.QtWidgets import QVBoxLayout
 from PyQt5.QtWidgets import QWidget
 
-from PyQt5.QtSql import QSqlDatabase
-from PyQt5.QtSql import QSqlQuery
-from PyQt5.QtSql import QSqlTableModel
-
 from beat.core.experiment import PROCESSOR_PREFIX
 from beat.core.experiment import EVALUATOR_PREFIX
 
@@ -64,6 +60,8 @@ from beat.backend.python.algorithm import Algorithm
 from ..backend.asset import Asset
 from ..backend.asset import AssetType
 from ..backend.assetmodel import AssetModel
+from ..backend.resourcemodels import ExperimentResources
+from ..backend.resourcemodels import AlgorithmResourceModel
 
 from ..decorators import frozen
 
@@ -240,230 +238,8 @@ class DatasetModel(QStringListModel):
 
 
 # ------------------------------------------------------------------------------
-# Prefix modelization
-
-
-class ExperimentResources:
-    """Modelization of the experiments resources"""
-
-    def __init__(self, context=None):
-        self.context = context
-
-        database = QSqlDatabase.addDatabase("QSQLITE")
-        database.setDatabaseName(":memory:")
-
-        if not database.open():
-            raise RuntimeError(
-                f"Failed to open database: {database.lastError().text()}"
-            )
-
-        self.refresh()
-
-    def setContext(self, context):
-        if self.context == context:
-            return
-
-        self.context = context
-        self.refresh()
-
-    def refresh(self):
-        ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
-        ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
-        INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
-        QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
-        QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
-        INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
-
-        query = QSqlQuery()
-
-        for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
-            if not query.exec_(query_str):
-                raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
-
-        for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
-            if not query.exec_(query_str):
-                raise RuntimeError(
-                    f"Failed to create table: {query.lastError().text()}"
-                )
-
-        if self.context is None:
-            return
-
-        prefix_path = self.context.meta["config"].path
-        model = AssetModel()
-        model.asset_type = AssetType.ALGORITHM
-        model.prefix_path = prefix_path
-        model.setLatestOnlyEnabled(False)
-
-        if not query.prepare(INSERT_ALGORITHM):
-            raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
-
-        for algorithm in model.stringList():
-            asset = Asset(prefix_path, AssetType.ALGORITHM, algorithm)
-            try:
-                declaration = asset.declaration
-            except json.JSONDecodeError:
-                continue
-
-            inputs = {}
-            outputs = {}
-            for group in declaration["groups"]:
-                inputs.update(group.get("inputs", {}))
-                outputs.update(group.get("outputs", {}))
-
-            query.addBindValue(algorithm)
-            query.addBindValue(declaration.get("type", "legacy"))
-            query.addBindValue(len(inputs))
-            query.addBindValue(len(outputs))
-            query.addBindValue("results" in declaration)
-
-            if not query.exec_():
-                raise RuntimeError(
-                    f"Failed to insert algorithm: {query.lastError().text()}"
-                )
-
-        if not query.prepare(INSERT_QUEUE):
-            raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
-
-        environments_path = self.context.meta["environments"]
-        with open(environments_path, "rt") as file:
-            environment_data = json.load(file)
-
-        for item in environment_data.get("remote", []):
-            env_name = item["name"]
-            env_version = item["version"]
-            # import ipdb; ipdb.set_trace()
-            for name in item["queues"].keys():
-                query.addBindValue(name)
-                query.addBindValue(env_name)
-                query.addBindValue(env_version)
-                query.addBindValue("remote")
-
-                if not query.exec_():
-                    raise RuntimeError(
-                        f"Failed to insert queue: {query.lastError().text()}"
-                    )
-
-        for _, image_info in environment_data.get("docker", {}).items():
-            env_name = image_info["name"]
-            env_version = image_info["version"]
-            query.addBindValue("Local")
-            query.addBindValue(env_name)
-            query.addBindValue(env_version)
-            query.addBindValue("docker")
-            if not query.exec_():
-                raise RuntimeError(
-                    f"Failed to insert queue: {query.lastError().text()}"
-                )
-
-
-class AlgorithmResourceModel(QSqlTableModel):
-    def __init__(self, parent=None):
-        super().__init__(parent=parent)
-
-        self._analyzer_enabled = False
-        self._input_count = None
-        self._output_count = None
-        self._types = []
-
-        self.setTable("algorithms")
-        self.select()
-        self.update()
-
-    def update(self):
-        filter_str = f"is_analyzer={self._analyzer_enabled}"
-        if self._input_count is not None:
-            filter_str += f" AND inputs={self._input_count}"
-        if self._output_count is not None:
-            filter_str += f" AND outputs={self._output_count}"
-
-        if self._types:
-            filter_str += " AND type in ({})".format(
-                ",".join([f"'{type_}'" for type_ in self._types])
-            )
-
-        self.setFilter(filter_str)
-
-    def setAnalyzerEnabled(self, enabled):
-        if self._analyzer_enabled == enabled:
-            return
-
-        self._analyzer_enabled = enabled
-        self.update()
-
-    def setInputCount(self, count):
-        if self._input_count == count:
-            return
-
-        self._input_count = count
-        self.update()
-
-    def setOutputCount(self, count):
-        if self._output_count == count:
-            return
-
-        self._output_count = count
-        self.update()
-
-    def setTypes(self, type_list):
-        if self._types == type_list:
-            return
-
-        self._types = type_list
-        self.update()
-
-
-class QueueResourceModel(QSqlTableModel):
-    def __init__(self, parent=None):
-        super().__init__(parent=parent)
-
-        self._environment = None
-        self._version = None
-        self._type = None
-
-        self.setTable("queues")
-        self.select()
-        self.update()
-
-    def update(self):
-        filter_str = ""
-        if self._environment is not None:
-            filter_str += f"env_name='{self._environment}'"
 
-        if self._version is not None:
-            if filter_str:
-                filter_str += " AND "
-            filter_str += f"env_version='{self._version}'"
 
-        if self._type is not None:
-            if filter_str:
-                filter_str += " AND "
-            filter_str += f"env_type='{self._type}'"
-
-        self.setFilter(filter_str)
-
-    def setEnvironment(self, name, version):
-        if self._environment == name and self._version == version:
-            return
-
-        self._environment = name
-        self._version = version
-        self.update()
-
-    def setType(self, type_):
-        if self._type == type_:
-            return
-
-        self._type = type_
-        self.update()
-
-    def dump(self):
-        print(self.filter())
-        for i in range(self.rowCount()):
-            print([self.index(i, j).data() for j in range(4)])
-
-
-# ------------------------------------------------------------------------------
 # Editors