Commit 26f28c19 authored by Flavio TARSETTI's avatar Flavio TARSETTI

[widgets][experimenteditor] removed prefix modelization for algorithms/experimenteditor

parent d7fda50e
...@@ -40,10 +40,13 @@ EDITED_KEY = "edited" ...@@ -40,10 +40,13 @@ EDITED_KEY = "edited"
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Prefix modelization # Prefix modelization
class ExperimentResources: class ExperimentResources:
"""Modelization of the experiments resources"""
def __init__(self, context=None): def __init__(self, context=None):
self.context = context self.context = context
...@@ -65,20 +68,27 @@ class ExperimentResources: ...@@ -65,20 +68,27 @@ class ExperimentResources:
self.refresh() self.refresh()
def refresh(self): def refresh(self):
if self.context is None:
return
ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms" ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)" ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)" INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
query = QSqlQuery() query = QSqlQuery()
if not query.exec_(ALGORITHM_TABLE_CLEANUP): for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}") if not query.exec_(query_str):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
if not query.exec_(ALGORITHM_TABLE): for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
raise RuntimeError(f"Failed to create table: {query.lastError().text()}") if not query.exec_(query_str):
raise RuntimeError(
f"Failed to create table: {query.lastError().text()}"
)
if self.context is None:
return
prefix_path = self.context.meta["config"].path prefix_path = self.context.meta["config"].path
model = AssetModel() model = AssetModel()
...@@ -113,6 +123,40 @@ class ExperimentResources: ...@@ -113,6 +123,40 @@ class ExperimentResources:
f"Failed to insert algorithm: {query.lastError().text()}" f"Failed to insert algorithm: {query.lastError().text()}"
) )
if not query.prepare(INSERT_QUEUE):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
environments_path = self.context.meta["environments"]
with open(environments_path, "rt") as file:
environment_data = json.load(file)
for item in environment_data.get("remote", []):
env_name = item["name"]
env_version = item["version"]
# import ipdb; ipdb.set_trace()
for name in item["queues"].keys():
query.addBindValue(name)
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("remote")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
for _, image_info in environment_data.get("docker", {}).items():
env_name = image_info["name"]
env_version = image_info["version"]
query.addBindValue("Local")
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("docker")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
class AlgorithmResourceModel(QSqlTableModel): class AlgorithmResourceModel(QSqlTableModel):
def __init__(self, parent=None): def __init__(self, parent=None):
...@@ -168,3 +212,55 @@ class AlgorithmResourceModel(QSqlTableModel): ...@@ -168,3 +212,55 @@ class AlgorithmResourceModel(QSqlTableModel):
self._types = type_list self._types = type_list
self.update() self.update()
class QueueResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._environment = None
self._version = None
self._type = None
self.setTable("queues")
self.select()
self.update()
def update(self):
filter_str = ""
if self._environment is not None:
filter_str += f"env_name='{self._environment}'"
if self._version is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_version='{self._version}'"
if self._type is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_type='{self._type}'"
self.setFilter(filter_str)
def setEnvironment(self, name, version):
if self._environment == name and self._version == version:
return
self._environment = name
self._version = version
self.update()
def setType(self, type_):
if self._type == type_:
return
self._type = type_
self.update()
def dump(self):
print(self.filter())
for i in range(self.rowCount()):
print([self.index(i, j).data() for j in range(4)])
...@@ -52,10 +52,6 @@ from PyQt5.QtWidgets import QTabWidget ...@@ -52,10 +52,6 @@ from PyQt5.QtWidgets import QTabWidget
from PyQt5.QtWidgets import QVBoxLayout from PyQt5.QtWidgets import QVBoxLayout
from PyQt5.QtWidgets import QWidget from PyQt5.QtWidgets import QWidget
from PyQt5.QtSql import QSqlDatabase
from PyQt5.QtSql import QSqlQuery
from PyQt5.QtSql import QSqlTableModel
from beat.core.experiment import PROCESSOR_PREFIX from beat.core.experiment import PROCESSOR_PREFIX
from beat.core.experiment import EVALUATOR_PREFIX from beat.core.experiment import EVALUATOR_PREFIX
...@@ -64,6 +60,8 @@ from beat.backend.python.algorithm import Algorithm ...@@ -64,6 +60,8 @@ from beat.backend.python.algorithm import Algorithm
from ..backend.asset import Asset from ..backend.asset import Asset
from ..backend.asset import AssetType from ..backend.asset import AssetType
from ..backend.assetmodel import AssetModel from ..backend.assetmodel import AssetModel
from ..backend.resourcemodels import ExperimentResources
from ..backend.resourcemodels import AlgorithmResourceModel
from ..decorators import frozen from ..decorators import frozen
...@@ -240,230 +238,8 @@ class DatasetModel(QStringListModel): ...@@ -240,230 +238,8 @@ class DatasetModel(QStringListModel):
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Prefix modelization
class ExperimentResources:
"""Modelization of the experiments resources"""
def __init__(self, context=None):
self.context = context
database = QSqlDatabase.addDatabase("QSQLITE")
database.setDatabaseName(":memory:")
if not database.open():
raise RuntimeError(
f"Failed to open database: {database.lastError().text()}"
)
self.refresh()
def setContext(self, context):
if self.context == context:
return
self.context = context
self.refresh()
def refresh(self):
ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
query = QSqlQuery()
for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
if not query.exec_(query_str):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
if not query.exec_(query_str):
raise RuntimeError(
f"Failed to create table: {query.lastError().text()}"
)
if self.context is None:
return
prefix_path = self.context.meta["config"].path
model = AssetModel()
model.asset_type = AssetType.ALGORITHM
model.prefix_path = prefix_path
model.setLatestOnlyEnabled(False)
if not query.prepare(INSERT_ALGORITHM):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
for algorithm in model.stringList():
asset = Asset(prefix_path, AssetType.ALGORITHM, algorithm)
try:
declaration = asset.declaration
except json.JSONDecodeError:
continue
inputs = {}
outputs = {}
for group in declaration["groups"]:
inputs.update(group.get("inputs", {}))
outputs.update(group.get("outputs", {}))
query.addBindValue(algorithm)
query.addBindValue(declaration.get("type", "legacy"))
query.addBindValue(len(inputs))
query.addBindValue(len(outputs))
query.addBindValue("results" in declaration)
if not query.exec_():
raise RuntimeError(
f"Failed to insert algorithm: {query.lastError().text()}"
)
if not query.prepare(INSERT_QUEUE):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
environments_path = self.context.meta["environments"]
with open(environments_path, "rt") as file:
environment_data = json.load(file)
for item in environment_data.get("remote", []):
env_name = item["name"]
env_version = item["version"]
# import ipdb; ipdb.set_trace()
for name in item["queues"].keys():
query.addBindValue(name)
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("remote")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
for _, image_info in environment_data.get("docker", {}).items():
env_name = image_info["name"]
env_version = image_info["version"]
query.addBindValue("Local")
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("docker")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
class AlgorithmResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._analyzer_enabled = False
self._input_count = None
self._output_count = None
self._types = []
self.setTable("algorithms")
self.select()
self.update()
def update(self):
filter_str = f"is_analyzer={self._analyzer_enabled}"
if self._input_count is not None:
filter_str += f" AND inputs={self._input_count}"
if self._output_count is not None:
filter_str += f" AND outputs={self._output_count}"
if self._types:
filter_str += " AND type in ({})".format(
",".join([f"'{type_}'" for type_ in self._types])
)
self.setFilter(filter_str)
def setAnalyzerEnabled(self, enabled):
if self._analyzer_enabled == enabled:
return
self._analyzer_enabled = enabled
self.update()
def setInputCount(self, count):
if self._input_count == count:
return
self._input_count = count
self.update()
def setOutputCount(self, count):
if self._output_count == count:
return
self._output_count = count
self.update()
def setTypes(self, type_list):
if self._types == type_list:
return
self._types = type_list
self.update()
class QueueResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._environment = None
self._version = None
self._type = None
self.setTable("queues")
self.select()
self.update()
def update(self):
filter_str = ""
if self._environment is not None:
filter_str += f"env_name='{self._environment}'"
if self._version is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_version='{self._version}'"
if self._type is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_type='{self._type}'"
self.setFilter(filter_str)
def setEnvironment(self, name, version):
if self._environment == name and self._version == version:
return
self._environment = name
self._version = version
self.update()
def setType(self, type_):
if self._type == type_:
return
self._type = type_
self.update()
def dump(self):
print(self.filter())
for i in range(self.rowCount()):
print([self.index(i, j).data() for j in range(4)])
# ------------------------------------------------------------------------------
# Editors # Editors
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment