[widgets][experimenteditor] Implement queue selection

The selection can be done for both remote and local
development.

beat/editor/test/conftest.py

@@ -95,8 +95,20 @@ def test_prefix():
             json.dumps(
                 {
                     "remote": [
-                        {"name": "Python 2.7", "version": "1.3.0"},
-                        {"name": "Python", "version": "2.0.0"},
+                        {
+                            "name": "Python 2.7",
+                            "version": "1.3.0",
+                            "queues": {"queue1": {}, "queue2": {}},
+                        },
+                        {
+                            "name": "Python",
+                            "version": "2.0.0",
+                            "queues": {
+                                "queue_extra_long": {},
+                                "queue_long": {},
+                                "queue_short": {},
+                            },
+                        },
                     ],
                     "docker": {
                         "Docker test env": {"name": "Pytorch", "version": "1.1.1"},

beat/editor/test/test_experimenteditor.py

@@ -27,7 +27,6 @@ import copy
 import pytest
 
 from PyQt5.QtCore import Qt
-from PyQt5.QtCore import QStringListModel
 
 from PyQt5.QtWidgets import QComboBox
 from PyQt5.QtWidgets import QCheckBox
@@ -62,6 +61,7 @@ from ..widgets.experimenteditor import typed_user_property
 
 from ..widgets.experimenteditor import ExperimentResources
 from ..widgets.experimenteditor import AlgorithmResourceModel
+from ..widgets.experimenteditor import QueueResourceModel
 
 from .conftest import prefix
 from .conftest import sync_prefix
@@ -287,6 +287,40 @@ class TestAlgorithmResourceModel:
         assert algorithm_model.rowCount() == query.value("cnt")
 
 
+class TestQueueResourceModel:
+    """Test the model used to generate suitable algorithm selections"""
+
+    @pytest.fixture
+    def prefix_model(self, beat_context):
+        return ExperimentResources(beat_context)
+
+    def test_default(self, prefix_model):
+        model = QueueResourceModel()
+
+        query = QSqlQuery()
+        assert query.exec_("SELECT COUNT(name) AS cnt FROM queues")
+
+        query.next()
+
+        assert model.rowCount() > 0
+        assert model.rowCount() == query.value("cnt")
+
+    def test_types(self, prefix_model):
+        model = QueueResourceModel()
+        for type_ in ["remote", "docker"]:
+            model.setType(type_)
+
+            query = QSqlQuery()
+            assert query.exec_(
+                f"SELECT COUNT(name) AS cnt FROM queues WHERE env_type='{type_}'"
+            )
+
+            query.next()
+
+            assert model.rowCount() > 0
+            assert model.rowCount() == query.value("cnt")
+
+
 class TestIOMapperDialog:
     """Test that the dialog used for input/output mapping works as expected"""
 
@@ -735,7 +769,6 @@ class TestExecutionPropertiesEditor(PropertiesEditorTestMixin, ParameterTestMixi
         editor = self.editor_klass(test_prefix)
         editor.setAlgorithmResourceModel(algorithm_model)
         editor.setEnvironmentModel(environment_model)
-        editor.setQueueModel(QStringListModel(["Test"]))
         return editor
 
 
@@ -752,7 +785,6 @@ class TestBlockEditor(TestExecutionPropertiesEditor):
 
         editor = self.editor_klass("block_name", test_prefix)
         editor.setEnvironmentModel(environment_model)
-        editor.setQueueModel(QStringListModel(["Test"]))
         return editor
 
 
@@ -769,7 +801,6 @@ class TestAnalyzerBlockEditor(PropertiesEditorTestMixin):
 
         editor = self.editor_klass("block_name", test_prefix)
         editor.setEnvironmentModel(environment_model)
-        editor.setQueueModel(QStringListModel(["Test"]))
         return editor
 
 
@@ -786,7 +817,6 @@ class TestLoopBlockEditor(TestExecutionPropertiesEditor):
 
         editor = self.editor_klass("block_name", test_prefix)
         editor.setEnvironmentModel(environment_model)
-        editor.setQueueModel(QStringListModel(["Test"]))
         return editor
 
     def test_edit_environment(
@@ -1030,7 +1060,6 @@ class TestGlobalParametersEditor:
         environment_model = EnvironmentModel()
         environment_model.setContext(beat_context)
         editor.setEnvironmentModel(environment_model)
-        editor.setQueueModel(QStringListModel(["Test", "Test2"]))
         return editor
 
     @pytest.mark.parametrize(
@@ -1080,7 +1109,8 @@ class TestGlobalParametersEditor:
     def test_change_queue(self, qtbot, gpe_editor, exp_globals):
         gpe_editor.load(exp_globals)
         with qtbot.waitSignal(gpe_editor.dataChanged):
-            gpe_editor.queue_combobox.setCurrentIndex(1)
+            index = gpe_editor.queue_combobox.currentIndex()
+            gpe_editor.queue_combobox.setCurrentIndex(change_index(index))
         assert gpe_editor.dump() != exp_globals
 
 

beat/editor/widgets/experimenteditor.py

@@ -117,7 +117,7 @@ class EnvironmentModel(QStandardItemModel):
     """Model wrapping the processing environment available"""
 
     def __init__(self, parent=None):
-        super().__init__(3, 0, parent)
+        super().__init__(4, 0, parent)
 
         self.context = None
 
@@ -134,6 +134,7 @@ class EnvironmentModel(QStandardItemModel):
                         QStandardItem(icon, visual_name),
                         QStandardItem(data["name"]),
                         QStandardItem(data["version"]),
+                        QStandardItem(type_),
                     ]
                 )
 
@@ -169,6 +170,14 @@ class EnvironmentModel(QStandardItemModel):
         version = self.data(version)
         return {"name": name, "version": version}
 
+    def environmentType(self, index):
+        if is_Qt_equal_or_higher("5.11"):
+            type_ = index.siblingAtColumn(3)
+        else:
+            type_ = index.sibling(index.row(), 3)
+
+        return self.data(type_)
+
 
 class ContainerWidget(ScrollWidget):
     """Container widget to show block editors"""
@@ -235,6 +244,8 @@ class DatasetModel(QStringListModel):
 
 
 class ExperimentResources:
+    """Modelization of the experiments resources"""
+
     def __init__(self, context=None):
         self.context = context
 
@@ -256,20 +267,27 @@ class ExperimentResources:
         self.refresh()
 
     def refresh(self):
-        if self.context is None:
-            return
-
         ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
         ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
         INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
+        QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
+        QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
+        INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
 
         query = QSqlQuery()
 
-        if not query.exec_(ALGORITHM_TABLE_CLEANUP):
-            raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
+        for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
+            if not query.exec_(query_str):
+                raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
 
-        if not query.exec_(ALGORITHM_TABLE):
-            raise RuntimeError(f"Failed to create table: {query.lastError().text()}")
+        for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
+            if not query.exec_(query_str):
+                raise RuntimeError(
+                    f"Failed to create table: {query.lastError().text()}"
+                )
+
+        if self.context is None:
+            return
 
         prefix_path = self.context.meta["config"].path
         model = AssetModel()
@@ -304,6 +322,40 @@ class ExperimentResources:
                     f"Failed to insert algorithm: {query.lastError().text()}"
                 )
 
+        if not query.prepare(INSERT_QUEUE):
+            raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
+
+        environments_path = self.context.meta["environments"]
+        with open(environments_path, "rt") as file:
+            environment_data = json.load(file)
+
+        for item in environment_data.get("remote", []):
+            env_name = item["name"]
+            env_version = item["version"]
+            # import ipdb; ipdb.set_trace()
+            for name in item["queues"].keys():
+                query.addBindValue(name)
+                query.addBindValue(env_name)
+                query.addBindValue(env_version)
+                query.addBindValue("remote")
+
+                if not query.exec_():
+                    raise RuntimeError(
+                        f"Failed to insert queue: {query.lastError().text()}"
+                    )
+
+        for _, image_info in environment_data.get("docker", {}).items():
+            env_name = image_info["name"]
+            env_version = image_info["version"]
+            query.addBindValue("Local")
+            query.addBindValue(env_name)
+            query.addBindValue(env_version)
+            query.addBindValue("docker")
+            if not query.exec_():
+                raise RuntimeError(
+                    f"Failed to insert queue: {query.lastError().text()}"
+                )
+
 
 class AlgorithmResourceModel(QSqlTableModel):
     def __init__(self, parent=None):
@@ -361,6 +413,56 @@ class AlgorithmResourceModel(QSqlTableModel):
         self.update()
 
 
+class QueueResourceModel(QSqlTableModel):
+    def __init__(self, parent=None):
+        super().__init__(parent=parent)
+
+        self._environment = None
+        self._version = None
+        self._type = None
+
+        self.setTable("queues")
+        self.select()
+        self.update()
+
+    def update(self):
+        filter_str = ""
+        if self._environment is not None:
+            filter_str += f"env_name='{self._environment}'"
+
+        if self._version is not None:
+            if filter_str:
+                filter_str += " AND "
+            filter_str += f"env_version='{self._version}'"
+
+        if self._type is not None:
+            if filter_str:
+                filter_str += " AND "
+            filter_str += f"env_type='{self._type}'"
+
+        self.setFilter(filter_str)
+
+    def setEnvironment(self, name, version):
+        if self._environment == name and self._version == version:
+            return
+
+        self._environment = name
+        self._version = version
+        self.update()
+
+    def setType(self, type_):
+        if self._type == type_:
+            return
+
+        self._type = type_
+        self.update()
+
+    def dump(self):
+        print(self.filter())
+        for i in range(self.rowCount()):
+            print([self.index(i, j).data() for j in range(4)])
+
+
 # ------------------------------------------------------------------------------
 # Editors
 
@@ -683,11 +785,13 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
         self._queue_enabled = True
         self.parameter_item = None
         self.algorithm_model = None
+        self.queue_model = QueueResourceModel()
         self.algorithm_combobox = QComboBox()
         self.algorithm_combobox.setObjectName("algorithms")
         self.environment_combobox = QComboBox()
         self.environment_combobox.setObjectName("environments")
         self.queue_combobox = QComboBox()
+        self.queue_combobox.setModel(self.queue_model)
         self.parameters_editor = AlgorithmParametersEditor(prefix_path)
 
         self.remap_button = QPushButton(QIcon(":/resources/remap"), "")
@@ -724,6 +828,9 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
         self.environment_combobox.currentIndexChanged.connect(
             lambda *args: setattr(self, "environment_changed", True)
         )
+        self.environment_combobox.currentIndexChanged.connect(
+            self.__onEnvironmentChanged
+        )
         self.queue_combobox.currentIndexChanged.connect(self.dataChanged)
         self.queue_combobox.currentIndexChanged.connect(
             lambda *args: setattr(self, "queue_changed", True)
@@ -765,6 +872,15 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
             self._queue_enabled
         )
 
+    @pyqtSlot(int)
+    def __onEnvironmentChanged(self, row):
+        model = self.environment_combobox.model()
+        index = model.index(row, 0)
+
+        self.queue_model.setEnvironment(**model.environment(index))
+        self.queue_model.setType(model.environmentType(index))
+        self.queue_combobox.setCurrentIndex(0)
+
     @pyqtSlot(str)
     def __onAlgorithmChanged(self, algorithm_name):
 
@@ -814,9 +930,6 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
     def environmentModel(self):
         return self.environment_combobox.model()
 
-    def setQueueModel(self, model):
-        self.queue_combobox.setModel(model)
-
     def setQueueEnabled(self, enabled):
         self._queue_enabled = enabled
         self.__updateUi()
@@ -905,11 +1018,6 @@ class AbstractBlockEditor(AbstractBaseEditor):
 
         raise NotImplementedError
 
-    def setQueueModel(self, model):
-        """Setup queue model"""
-
-        raise NotImplementedError
-
 
 class BlockEditor(AbstractBlockEditor):
     def __init__(self, block_name, prefix_path, parent=None):
@@ -942,9 +1050,6 @@ class BlockEditor(AbstractBlockEditor):
     def environmentModel(self):
         return self.properties_editor.environmentModel()
 
-    def setQueueModel(self, model):
-        self.properties_editor.setQueueModel(model)
-
     def load(self, json_object):
         self.properties_editor.load(json_object)
 
@@ -1019,9 +1124,6 @@ class LoopBlockEditor(AbstractBlockEditor):
     def environmentModel(self):
         return self.processor_properties_editor.environmentModel()
 
-    def setQueueModel(self, model):
-        self.processor_properties_editor.setQueueModel(model)
-
     def load(self, json_object):
         processor_data = {
             key[len(PROCESSOR_PREFIX) :]: value
@@ -1087,9 +1189,11 @@ class GlobalParametersEditor(AbstractBaseEditor):
         self.json_object = {}
         self.environment_changed = False
         self.queue_changed = False
+        self.queue_model = QueueResourceModel()
 
         self.environment_combobox = QComboBox()
         self.queue_combobox = QComboBox()
+        self.queue_combobox.setModel(self.queue_model)
         self.parameters_editor_listwidget = EditorListWidget()
 
         form_layout = QFormLayout()
@@ -1105,18 +1209,29 @@ class GlobalParametersEditor(AbstractBaseEditor):
         self.environment_combobox.currentIndexChanged.connect(
             lambda *args: setattr(self, "environment_changed", True)
         )
+        self.environment_combobox.currentIndexChanged.connect(
+            self.__onEnvironmentChanged
+        )
         self.queue_combobox.currentIndexChanged.connect(self.dataChanged)
         self.queue_combobox.currentIndexChanged.connect(
             lambda *args: setattr(self, "queue_changed", True)
         )
+
         self.parameters_editor_listwidget.dataChanged.connect(self.dataChanged)
+        self.prefixPathChanged.connect(self.queue_model.select)
+
+    @pyqtSlot(int)
+    def __onEnvironmentChanged(self, row):
+        model = self.environment_combobox.model()
+        index = model.index(row, 0)
+
+        self.queue_model.setEnvironment(**model.environment(index))
+        self.queue_model.setType(model.environmentType(index))
+        self.queue_combobox.setCurrentIndex(0)
 
     def setEnvironmentModel(self, model):
         self.environment_combobox.setModel(model)
 
-    def setQueueModel(self, model):
-        self.queue_combobox.setModel(model)
-
     def clear(self):
         self.environment_combobox.setCurrentIndex(0)
         self.parameters_editor_listwidget.clear()
@@ -1204,11 +1319,9 @@ class ExperimentEditor(AbstractAssetEditor):
         self.setObjectName(self.__class__.__name__)
         self.set_title(self.tr("Experiment"))
 
-        self.prefix_model = ExperimentResources()
+        self.resource_model = ExperimentResources()
 
         self.processing_env_model = EnvironmentModel()
-        self.queue_model = QStringListModel()
-        self.queue_model.setStringList(["Local"])
 
         self.dataset_model = DatasetModel()
 
@@ -1222,7 +1335,6 @@ class ExperimentEditor(AbstractAssetEditor):
         self.analyzers_widget = ContainerWidget()
         self.globalparameters_widget = GlobalParametersEditor(self.prefixPath())
         self.globalparameters_widget.setEnvironmentModel(self.processing_env_model)
-        self.globalparameters_widget.setQueueModel(self.queue_model)
 
         self.tabwidget = QTabWidget()
         self.tabwidget.addTab(self.datasets_widget, self.tr("Datasets"))
@@ -1250,7 +1362,7 @@ class ExperimentEditor(AbstractAssetEditor):
 
     @pyqtSlot()
     def __update(self):
-        self.prefix_model.setContext(self.context)
+        self.resource_model.setContext(self.context)
 
         for object_ in [
             self.algorithm_model,
@@ -1396,7 +1508,7 @@ class ExperimentEditor(AbstractAssetEditor):
     def _load_json(self, json_object):
         """Load the json object passed as parameter"""
 
-        self.prefix_model.refresh()
+        self.resource_model.refresh()
 
         for widget in [
             self.datasets_widget,
@@ -1427,7 +1539,6 @@ class ExperimentEditor(AbstractAssetEditor):
                 for name, data in items.items():
                     editor = editor_klass(name, self.prefix_path)
                     editor.setEnvironmentModel(self.processing_env_model)
-                    editor.setQueueModel(self.queue_model)
                     editor.load(data)
                     editor.algorithmChanged.connect(self.__onAlgorithmChanged)
                     container.addWidget(editor)