Commit 26f28c19 authored by Flavio TARSETTI's avatar Flavio TARSETTI

[widgets][experimenteditor] removed prefix modelization for algorithms/experimenteditor

parent d7fda50e
......@@ -40,10 +40,13 @@ EDITED_KEY = "edited"
# ------------------------------------------------------------------------------
# Prefix modelization
class ExperimentResources:
"""Modelization of the experiments resources"""
def __init__(self, context=None):
self.context = context
......@@ -65,20 +68,27 @@ class ExperimentResources:
self.refresh()
def refresh(self):
if self.context is None:
return
ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
query = QSqlQuery()
if not query.exec_(ALGORITHM_TABLE_CLEANUP):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
if not query.exec_(query_str):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
if not query.exec_(ALGORITHM_TABLE):
raise RuntimeError(f"Failed to create table: {query.lastError().text()}")
for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
if not query.exec_(query_str):
raise RuntimeError(
f"Failed to create table: {query.lastError().text()}"
)
if self.context is None:
return
prefix_path = self.context.meta["config"].path
model = AssetModel()
......@@ -113,6 +123,40 @@ class ExperimentResources:
f"Failed to insert algorithm: {query.lastError().text()}"
)
if not query.prepare(INSERT_QUEUE):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
environments_path = self.context.meta["environments"]
with open(environments_path, "rt") as file:
environment_data = json.load(file)
for item in environment_data.get("remote", []):
env_name = item["name"]
env_version = item["version"]
# import ipdb; ipdb.set_trace()
for name in item["queues"].keys():
query.addBindValue(name)
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("remote")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
for _, image_info in environment_data.get("docker", {}).items():
env_name = image_info["name"]
env_version = image_info["version"]
query.addBindValue("Local")
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("docker")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
class AlgorithmResourceModel(QSqlTableModel):
def __init__(self, parent=None):
......@@ -168,3 +212,55 @@ class AlgorithmResourceModel(QSqlTableModel):
self._types = type_list
self.update()
class QueueResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._environment = None
self._version = None
self._type = None
self.setTable("queues")
self.select()
self.update()
def update(self):
filter_str = ""
if self._environment is not None:
filter_str += f"env_name='{self._environment}'"
if self._version is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_version='{self._version}'"
if self._type is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_type='{self._type}'"
self.setFilter(filter_str)
def setEnvironment(self, name, version):
if self._environment == name and self._version == version:
return
self._environment = name
self._version = version
self.update()
def setType(self, type_):
if self._type == type_:
return
self._type = type_
self.update()
def dump(self):
print(self.filter())
for i in range(self.rowCount()):
print([self.index(i, j).data() for j in range(4)])
......@@ -52,10 +52,6 @@ from PyQt5.QtWidgets import QTabWidget
from PyQt5.QtWidgets import QVBoxLayout
from PyQt5.QtWidgets import QWidget
from PyQt5.QtSql import QSqlDatabase
from PyQt5.QtSql import QSqlQuery
from PyQt5.QtSql import QSqlTableModel
from beat.core.experiment import PROCESSOR_PREFIX
from beat.core.experiment import EVALUATOR_PREFIX
......@@ -64,6 +60,8 @@ from beat.backend.python.algorithm import Algorithm
from ..backend.asset import Asset
from ..backend.asset import AssetType
from ..backend.assetmodel import AssetModel
from ..backend.resourcemodels import ExperimentResources
from ..backend.resourcemodels import AlgorithmResourceModel
from ..decorators import frozen
......@@ -240,230 +238,8 @@ class DatasetModel(QStringListModel):
# ------------------------------------------------------------------------------
# Prefix modelization
class ExperimentResources:
"""Modelization of the experiments resources"""
def __init__(self, context=None):
self.context = context
database = QSqlDatabase.addDatabase("QSQLITE")
database.setDatabaseName(":memory:")
if not database.open():
raise RuntimeError(
f"Failed to open database: {database.lastError().text()}"
)
self.refresh()
def setContext(self, context):
if self.context == context:
return
self.context = context
self.refresh()
def refresh(self):
ALGORITHM_TABLE_CLEANUP = "DROP TABLE IF EXISTS algorithms"
ALGORITHM_TABLE = "CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
INSERT_ALGORITHM = "INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
QUEUE_TABLE_CLEANUP = "DROP TABLE IF EXISTS queues"
QUEUE_TABLE = "CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
INSERT_QUEUE = "INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
query = QSqlQuery()
for query_str in [ALGORITHM_TABLE_CLEANUP, QUEUE_TABLE_CLEANUP]:
if not query.exec_(query_str):
raise RuntimeError(f"Failed to drop table: {query.lastError().text()}")
for query_str in [ALGORITHM_TABLE, QUEUE_TABLE]:
if not query.exec_(query_str):
raise RuntimeError(
f"Failed to create table: {query.lastError().text()}"
)
if self.context is None:
return
prefix_path = self.context.meta["config"].path
model = AssetModel()
model.asset_type = AssetType.ALGORITHM
model.prefix_path = prefix_path
model.setLatestOnlyEnabled(False)
if not query.prepare(INSERT_ALGORITHM):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
for algorithm in model.stringList():
asset = Asset(prefix_path, AssetType.ALGORITHM, algorithm)
try:
declaration = asset.declaration
except json.JSONDecodeError:
continue
inputs = {}
outputs = {}
for group in declaration["groups"]:
inputs.update(group.get("inputs", {}))
outputs.update(group.get("outputs", {}))
query.addBindValue(algorithm)
query.addBindValue(declaration.get("type", "legacy"))
query.addBindValue(len(inputs))
query.addBindValue(len(outputs))
query.addBindValue("results" in declaration)
if not query.exec_():
raise RuntimeError(
f"Failed to insert algorithm: {query.lastError().text()}"
)
if not query.prepare(INSERT_QUEUE):
raise RuntimeError(f"Failed to prepare query: {query.lastError().text()}")
environments_path = self.context.meta["environments"]
with open(environments_path, "rt") as file:
environment_data = json.load(file)
for item in environment_data.get("remote", []):
env_name = item["name"]
env_version = item["version"]
# import ipdb; ipdb.set_trace()
for name in item["queues"].keys():
query.addBindValue(name)
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("remote")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
for _, image_info in environment_data.get("docker", {}).items():
env_name = image_info["name"]
env_version = image_info["version"]
query.addBindValue("Local")
query.addBindValue(env_name)
query.addBindValue(env_version)
query.addBindValue("docker")
if not query.exec_():
raise RuntimeError(
f"Failed to insert queue: {query.lastError().text()}"
)
class AlgorithmResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._analyzer_enabled = False
self._input_count = None
self._output_count = None
self._types = []
self.setTable("algorithms")
self.select()
self.update()
def update(self):
filter_str = f"is_analyzer={self._analyzer_enabled}"
if self._input_count is not None:
filter_str += f" AND inputs={self._input_count}"
if self._output_count is not None:
filter_str += f" AND outputs={self._output_count}"
if self._types:
filter_str += " AND type in ({})".format(
",".join([f"'{type_}'" for type_ in self._types])
)
self.setFilter(filter_str)
def setAnalyzerEnabled(self, enabled):
if self._analyzer_enabled == enabled:
return
self._analyzer_enabled = enabled
self.update()
def setInputCount(self, count):
if self._input_count == count:
return
self._input_count = count
self.update()
def setOutputCount(self, count):
if self._output_count == count:
return
self._output_count = count
self.update()
def setTypes(self, type_list):
if self._types == type_list:
return
self._types = type_list
self.update()
class QueueResourceModel(QSqlTableModel):
def __init__(self, parent=None):
super().__init__(parent=parent)
self._environment = None
self._version = None
self._type = None
self.setTable("queues")
self.select()
self.update()
def update(self):
filter_str = ""
if self._environment is not None:
filter_str += f"env_name='{self._environment}'"
if self._version is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_version='{self._version}'"
if self._type is not None:
if filter_str:
filter_str += " AND "
filter_str += f"env_type='{self._type}'"
self.setFilter(filter_str)
def setEnvironment(self, name, version):
if self._environment == name and self._version == version:
return
self._environment = name
self._version = version
self.update()
def setType(self, type_):
if self._type == type_:
return
self._type = type_
self.update()
def dump(self):
print(self.filter())
for i in range(self.rowCount()):
print([self.index(i, j).data() for j in range(4)])
# ------------------------------------------------------------------------------
# Editors
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment