Commit 50dfd546 authored by Samuel GAIST's avatar Samuel GAIST Committed by Samuel GAIST
Browse files

[widgets][experimenteditor] Implement queue selection

The selection can be done for both remote and local
development.
parent 379733a7
Pipeline #35414 passed with stage
in 5 minutes and 36 seconds
......@@ -95,8 +95,20 @@ def test_prefix():
json.dumps(
{
"remote": [
{"name": "Python 2.7", "version": "1.3.0"},
{"name": "Python", "version": "2.0.0"},
{
"name": "Python 2.7",
"version": "1.3.0",
"queues": {"queue1": {}, "queue2": {}},
},
{
"name": "Python",
"version": "2.0.0",
"queues": {
"queue_extra_long": {},
"queue_long": {},
"queue_short": {},
},
},
],
"docker": {
"Docker test env": {"name": "Pytorch", "version": "1.1.1"},
......
......@@ -27,7 +27,6 @@ import copy
import pytest
from PyQt5.QtCore import Qt
from PyQt5.QtCore import QStringListModel
from PyQt5.QtWidgets import QComboBox
from PyQt5.QtWidgets import QCheckBox
......@@ -62,6 +61,7 @@ from ..widgets.experimenteditor import typed_user_property
from ..widgets.experimenteditor import ExperimentResources
from ..widgets.experimenteditor import AlgorithmResourceModel
from ..widgets.experimenteditor import QueueResourceModel
from .conftest import prefix
from .conftest import sync_prefix
......@@ -287,6 +287,40 @@ class TestAlgorithmResourceModel:
assert algorithm_model.rowCount() == query.value("cnt")
class TestQueueResourceModel:
"""Test the model used to generate suitable algorithm selections"""
@pytest.fixture
def prefix_model(self, beat_context):
return ExperimentResources(beat_context)
def test_default(self, prefix_model):
model = QueueResourceModel()
query = QSqlQuery()
assert query.exec_("SELECT COUNT(name) AS cnt FROM queues")
query.next()
assert model.rowCount() > 0
assert model.rowCount() == query.value("cnt")
def test_types(self, prefix_model):
model = QueueResourceModel()
for type_ in ["remote", "docker"]:
model.setType(type_)
query = QSqlQuery()
assert query.exec_(
f"SELECT COUNT(name) AS cnt FROM queues WHERE env_type='{type_}'"
)
query.next()
assert model.rowCount() > 0
assert model.rowCount() == query.value("cnt")
class TestIOMapperDialog:
"""Test that the dialog used for input/output mapping works as expected"""
......@@ -735,7 +769,6 @@ class TestExecutionPropertiesEditor(PropertiesEditorTestMixin, ParameterTestMixi
editor = self.editor_klass(test_prefix)
editor.setAlgorithmResourceModel(algorithm_model)
editor.setEnvironmentModel(environment_model)
editor.setQueueModel(QStringListModel(["Test"]))
return editor
......@@ -752,7 +785,6 @@ class TestBlockEditor(TestExecutionPropertiesEditor):
editor = self.editor_klass("block_name", test_prefix)
editor.setEnvironmentModel(environment_model)
editor.setQueueModel(QStringListModel(["Test"]))
return editor
......@@ -769,7 +801,6 @@ class TestAnalyzerBlockEditor(PropertiesEditorTestMixin):
editor = self.editor_klass("block_name", test_prefix)
editor.setEnvironmentModel(environment_model)
editor.setQueueModel(QStringListModel(["Test"]))
return editor
......@@ -786,7 +817,6 @@ class TestLoopBlockEditor(TestExecutionPropertiesEditor):
editor = self.editor_klass("block_name", test_prefix)
editor.setEnvironmentModel(environment_model)
editor.setQueueModel(QStringListModel(["Test"]))
return editor
def test_edit_environment(
......@@ -1030,7 +1060,6 @@ class TestGlobalParametersEditor:
environment_model = EnvironmentModel()
environment_model.setContext(beat_context)
editor.setEnvironmentModel(environment_model)
editor.setQueueModel(QStringListModel(["Test", "Test2"]))
return editor
@pytest.mark.parametrize(
......@@ -1080,7 +1109,8 @@ class TestGlobalParametersEditor:
def test_change_queue(self, qtbot, gpe_editor, exp_globals):
gpe_editor.load(exp_globals)
with qtbot.waitSignal(gpe_editor.dataChanged):
gpe_editor.queue_combobox.setCurrentIndex(1)
index = gpe_editor.queue_combobox.currentIndex()
gpe_editor.queue_combobox.setCurrentIndex(change_index(index))
assert gpe_editor.dump() != exp_globals
......
......@@ -117,7 +117,7 @@ class EnvironmentModel(QStandardItemModel):
"""Model wrapping the processing environment available"""
def __init__(self, parent=None):
super().__init__(3, 0, parent)
super().__init__(4, 0, parent)
self.context = None
......@@ -134,6 +134,7 @@ class EnvironmentModel(QStandardItemModel):
QStandardItem(icon, visual_name),
QStandardItem(data["name"]),
QStandardItem(data["version"]),
QStandardItem(type_),
]
)
......@@ -169,6 +170,14 @@ class EnvironmentModel(QStandardItemModel):
version = self.data(version)
return {"name": name, "version": version}
def environmentType(self, index):
if is_Qt_equal_or_higher("5.11"):
type_ = index.siblingAtColumn(3)
else:
type_ = index.sibling(index.row(), 3)
return self.data(type_)
class ContainerWidget(ScrollWidget):
"""Container widget to show block editors"""
......@@ -235,6 +244,8 @@ class DatasetModel(QStringListModel):
class ExperimentResources:
"""Modelization of the experiments resources"""
def __init__(self, context=None):
self.context = context
......@@ -256,20 +267,27 @@ class ExperimentResources:
self.refresh()
def refresh(self):
if self.context is None:
return
ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
query = QSqlQuery()
if not query.exec_(ALGORITHM_TABLE_CLEANUP):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
if not query.exec_(query_str):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
if not query.exec_(ALGORITHM_TABLE):
raise RuntimeError(f"Failed to create table: {query.lastError().text()}")
for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
if not query.exec_(query_str):
raise RuntimeError(
f"Failed to create table: {query.lastError().text()}"
)
if self.context is None:
return
prefix_path = self.context.meta["config"].path
model = AssetModel()
......@@ -304,6 +322,40 @@ class ExperimentResources:
f"Failed to insert algorithm: {query.lastError().text()}"
)
if not query.prepare(INSERT_QUEUE):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
environments_path = self.context.meta["environments"]
with open(environments_path, "rt") as file:
environment_data = json.load(file)
for item in environment_data.get("remote", []):
env_name = item["name"]
env_version = item["version"]
# import ipdb; ipdb.set_trace()
for name in item["queues"].keys():
query.addBindValue(name)
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("remote")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
for _, image_info in environment_data.get("docker", {}).items():
env_name = image_info["name"]
env_version = image_info["version"]
query.addBindValue("Local")
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("docker")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
class AlgorithmResourceModel(QSqlTableModel):
def __init__(self, parent=None):
......@@ -361,6 +413,56 @@ class AlgorithmResourceModel(QSqlTableModel):
self.update()
class QueueResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._environment = None
self._version = None
self._type = None
self.setTable("queues")
self.select()
self.update()
def update(self):
filter_str = ""
if self._environment is not None:
filter_str += f"env_name='{self._environment}'"
if self._version is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_version='{self._version}'"
if self._type is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_type='{self._type}'"
self.setFilter(filter_str)
def setEnvironment(self, name, version):
if self._environment == name and self._version == version:
return
self._environment = name
self._version = version
self.update()
def setType(self, type_):
if self._type == type_:
return
self._type = type_
self.update()
def dump(self):
print(self.filter())
for i in range(self.rowCount()):
print([self.index(i, j).data() for j in range(4)])
# ------------------------------------------------------------------------------
# Editors
......@@ -683,11 +785,13 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
self._queue_enabled = True
self.parameter_item = None
self.algorithm_model = None
self.queue_model = QueueResourceModel()
self.algorithm_combobox = QComboBox()
self.algorithm_combobox.setObjectName("algorithms")
self.environment_combobox = QComboBox()
self.environment_combobox.setObjectName("environments")
self.queue_combobox = QComboBox()
self.queue_combobox.setModel(self.queue_model)
self.parameters_editor = AlgorithmParametersEditor(prefix_path)
self.remap_button = QPushButton(QIcon(":/resources/remap"), "")
......@@ -724,6 +828,9 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
self.environment_combobox.currentIndexChanged.connect(
lambda *args: setattr(self, "environment_changed", True)
)
self.environment_combobox.currentIndexChanged.connect(
self.__onEnvironmentChanged
)
self.queue_combobox.currentIndexChanged.connect(self.dataChanged)
self.queue_combobox.currentIndexChanged.connect(
lambda *args: setattr(self, "queue_changed", True)
......@@ -765,6 +872,15 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
self._queue_enabled
)
@pyqtSlot(int)
def __onEnvironmentChanged(self, row):
model = self.environment_combobox.model()
index = model.index(row, 0)
self.queue_model.setEnvironment(**model.environment(index))
self.queue_model.setType(model.environmentType(index))
self.queue_combobox.setCurrentIndex(0)
@pyqtSlot(str)
def __onAlgorithmChanged(self, algorithm_name):
......@@ -814,9 +930,6 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
def environmentModel(self):
return self.environment_combobox.model()
def setQueueModel(self, model):
self.queue_combobox.setModel(model)
def setQueueEnabled(self, enabled):
self._queue_enabled = enabled
self.__updateUi()
......@@ -905,11 +1018,6 @@ class AbstractBlockEditor(AbstractBaseEditor):
raise NotImplementedError
def setQueueModel(self, model):
"""Setup queue model"""
raise NotImplementedError
class BlockEditor(AbstractBlockEditor):
def __init__(self, block_name, prefix_path, parent=None):
......@@ -942,9 +1050,6 @@ class BlockEditor(AbstractBlockEditor):
def environmentModel(self):
return self.properties_editor.environmentModel()
def setQueueModel(self, model):
self.properties_editor.setQueueModel(model)
def load(self, json_object):
self.properties_editor.load(json_object)
......@@ -1019,9 +1124,6 @@ class LoopBlockEditor(AbstractBlockEditor):
def environmentModel(self):
return self.processor_properties_editor.environmentModel()
def setQueueModel(self, model):
self.processor_properties_editor.setQueueModel(model)
def load(self, json_object):
processor_data = {
key[len(PROCESSOR_PREFIX) :]: value
......@@ -1087,9 +1189,11 @@ class GlobalParametersEditor(AbstractBaseEditor):
self.json_object = {}
self.environment_changed = False
self.queue_changed = False
self.queue_model = QueueResourceModel()
self.environment_combobox = QComboBox()
self.queue_combobox = QComboBox()
self.queue_combobox.setModel(self.queue_model)
self.parameters_editor_listwidget = EditorListWidget()
form_layout = QFormLayout()
......@@ -1105,18 +1209,29 @@ class GlobalParametersEditor(AbstractBaseEditor):
self.environment_combobox.currentIndexChanged.connect(
lambda *args: setattr(self, "environment_changed", True)
)
self.environment_combobox.currentIndexChanged.connect(
self.__onEnvironmentChanged
)
self.queue_combobox.currentIndexChanged.connect(self.dataChanged)
self.queue_combobox.currentIndexChanged.connect(
lambda *args: setattr(self, "queue_changed", True)
)
self.parameters_editor_listwidget.dataChanged.connect(self.dataChanged)
self.prefixPathChanged.connect(self.queue_model.select)
@pyqtSlot(int)
def __onEnvironmentChanged(self, row):
model = self.environment_combobox.model()
index = model.index(row, 0)
self.queue_model.setEnvironment(**model.environment(index))
self.queue_model.setType(model.environmentType(index))
self.queue_combobox.setCurrentIndex(0)
def setEnvironmentModel(self, model):
self.environment_combobox.setModel(model)
def setQueueModel(self, model):
self.queue_combobox.setModel(model)
def clear(self):
self.environment_combobox.setCurrentIndex(0)
self.parameters_editor_listwidget.clear()
......@@ -1204,11 +1319,9 @@ class ExperimentEditor(AbstractAssetEditor):
self.setObjectName(self.__class__.__name__)
self.set_title(self.tr("Experiment"))
self.prefix_model = ExperimentResources()
self.resource_model = ExperimentResources()
self.processing_env_model = EnvironmentModel()
self.queue_model = QStringListModel()
self.queue_model.setStringList(["Local"])
self.dataset_model = DatasetModel()
......@@ -1222,7 +1335,6 @@ class ExperimentEditor(AbstractAssetEditor):
self.analyzers_widget = ContainerWidget()
self.globalparameters_widget = GlobalParametersEditor(self.prefixPath())
self.globalparameters_widget.setEnvironmentModel(self.processing_env_model)
self.globalparameters_widget.setQueueModel(self.queue_model)
self.tabwidget = QTabWidget()
self.tabwidget.addTab(self.datasets_widget, self.tr("Datasets"))
......@@ -1250,7 +1362,7 @@ class ExperimentEditor(AbstractAssetEditor):
@pyqtSlot()
def __update(self):
self.prefix_model.setContext(self.context)
self.resource_model.setContext(self.context)
for object_ in [
self.algorithm_model,
......@@ -1396,7 +1508,7 @@ class ExperimentEditor(AbstractAssetEditor):
def _load_json(self, json_object):
"""Load the json object passed as parameter"""
self.prefix_model.refresh()
self.resource_model.refresh()
for widget in [
self.datasets_widget,
......@@ -1427,7 +1539,6 @@ class ExperimentEditor(AbstractAssetEditor):
for name, data in items.items():
editor = editor_klass(name, self.prefix_path)
editor.setEnvironmentModel(self.processing_env_model)
editor.setQueueModel(self.queue_model)
editor.load(data)
editor.algorithmChanged.connect(self.__onAlgorithmChanged)
container.addWidget(editor)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment