Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.editor
Commits
50dfd546
Commit
50dfd546
authored
Oct 10, 2019
by
Samuel GAIST
Committed by
Samuel GAIST
Nov 15, 2019
Browse files
[widgets][experimenteditor] Implement queue selection
The selection can be done for both remote and local development.
parent
379733a7
Pipeline
#35414
passed with stage
in 5 minutes and 36 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
beat/editor/test/conftest.py
View file @
50dfd546
...
...
@@ -95,8 +95,20 @@ def test_prefix():
json
.
dumps
(
{
"remote"
:
[
{
"name"
:
"Python 2.7"
,
"version"
:
"1.3.0"
},
{
"name"
:
"Python"
,
"version"
:
"2.0.0"
},
{
"name"
:
"Python 2.7"
,
"version"
:
"1.3.0"
,
"queues"
:
{
"queue1"
:
{},
"queue2"
:
{}},
},
{
"name"
:
"Python"
,
"version"
:
"2.0.0"
,
"queues"
:
{
"queue_extra_long"
:
{},
"queue_long"
:
{},
"queue_short"
:
{},
},
},
],
"docker"
:
{
"Docker test env"
:
{
"name"
:
"Pytorch"
,
"version"
:
"1.1.1"
},
...
...
beat/editor/test/test_experimenteditor.py
View file @
50dfd546
...
...
@@ -27,7 +27,6 @@ import copy
import
pytest
from
PyQt5.QtCore
import
Qt
from
PyQt5.QtCore
import
QStringListModel
from
PyQt5.QtWidgets
import
QComboBox
from
PyQt5.QtWidgets
import
QCheckBox
...
...
@@ -62,6 +61,7 @@ from ..widgets.experimenteditor import typed_user_property
from
..widgets.experimenteditor
import
ExperimentResources
from
..widgets.experimenteditor
import
AlgorithmResourceModel
from
..widgets.experimenteditor
import
QueueResourceModel
from
.conftest
import
prefix
from
.conftest
import
sync_prefix
...
...
@@ -287,6 +287,40 @@ class TestAlgorithmResourceModel:
assert
algorithm_model
.
rowCount
()
==
query
.
value
(
"cnt"
)
class
TestQueueResourceModel
:
"""Test the model used to generate suitable algorithm selections"""
@
pytest
.
fixture
def
prefix_model
(
self
,
beat_context
):
return
ExperimentResources
(
beat_context
)
def
test_default
(
self
,
prefix_model
):
model
=
QueueResourceModel
()
query
=
QSqlQuery
()
assert
query
.
exec_
(
"SELECT COUNT(name) AS cnt FROM queues"
)
query
.
next
()
assert
model
.
rowCount
()
>
0
assert
model
.
rowCount
()
==
query
.
value
(
"cnt"
)
def
test_types
(
self
,
prefix_model
):
model
=
QueueResourceModel
()
for
type_
in
[
"remote"
,
"docker"
]:
model
.
setType
(
type_
)
query
=
QSqlQuery
()
assert
query
.
exec_
(
f
"SELECT COUNT(name) AS cnt FROM queues WHERE env_type='
{
type_
}
'"
)
query
.
next
()
assert
model
.
rowCount
()
>
0
assert
model
.
rowCount
()
==
query
.
value
(
"cnt"
)
class
TestIOMapperDialog
:
"""Test that the dialog used for input/output mapping works as expected"""
...
...
@@ -735,7 +769,6 @@ class TestExecutionPropertiesEditor(PropertiesEditorTestMixin, ParameterTestMixi
editor
=
self
.
editor_klass
(
test_prefix
)
editor
.
setAlgorithmResourceModel
(
algorithm_model
)
editor
.
setEnvironmentModel
(
environment_model
)
editor
.
setQueueModel
(
QStringListModel
([
"Test"
]))
return
editor
...
...
@@ -752,7 +785,6 @@ class TestBlockEditor(TestExecutionPropertiesEditor):
editor
=
self
.
editor_klass
(
"block_name"
,
test_prefix
)
editor
.
setEnvironmentModel
(
environment_model
)
editor
.
setQueueModel
(
QStringListModel
([
"Test"
]))
return
editor
...
...
@@ -769,7 +801,6 @@ class TestAnalyzerBlockEditor(PropertiesEditorTestMixin):
editor
=
self
.
editor_klass
(
"block_name"
,
test_prefix
)
editor
.
setEnvironmentModel
(
environment_model
)
editor
.
setQueueModel
(
QStringListModel
([
"Test"
]))
return
editor
...
...
@@ -786,7 +817,6 @@ class TestLoopBlockEditor(TestExecutionPropertiesEditor):
editor
=
self
.
editor_klass
(
"block_name"
,
test_prefix
)
editor
.
setEnvironmentModel
(
environment_model
)
editor
.
setQueueModel
(
QStringListModel
([
"Test"
]))
return
editor
def
test_edit_environment
(
...
...
@@ -1030,7 +1060,6 @@ class TestGlobalParametersEditor:
environment_model
=
EnvironmentModel
()
environment_model
.
setContext
(
beat_context
)
editor
.
setEnvironmentModel
(
environment_model
)
editor
.
setQueueModel
(
QStringListModel
([
"Test"
,
"Test2"
]))
return
editor
@
pytest
.
mark
.
parametrize
(
...
...
@@ -1080,7 +1109,8 @@ class TestGlobalParametersEditor:
def
test_change_queue
(
self
,
qtbot
,
gpe_editor
,
exp_globals
):
gpe_editor
.
load
(
exp_globals
)
with
qtbot
.
waitSignal
(
gpe_editor
.
dataChanged
):
gpe_editor
.
queue_combobox
.
setCurrentIndex
(
1
)
index
=
gpe_editor
.
queue_combobox
.
currentIndex
()
gpe_editor
.
queue_combobox
.
setCurrentIndex
(
change_index
(
index
))
assert
gpe_editor
.
dump
()
!=
exp_globals
...
...
beat/editor/widgets/experimenteditor.py
View file @
50dfd546
...
...
@@ -117,7 +117,7 @@ class EnvironmentModel(QStandardItemModel):
"""Model wrapping the processing environment available"""
def
__init__
(
self
,
parent
=
None
):
super
().
__init__
(
3
,
0
,
parent
)
super
().
__init__
(
4
,
0
,
parent
)
self
.
context
=
None
...
...
@@ -134,6 +134,7 @@ class EnvironmentModel(QStandardItemModel):
QStandardItem
(
icon
,
visual_name
),
QStandardItem
(
data
[
"name"
]),
QStandardItem
(
data
[
"version"
]),
QStandardItem
(
type_
),
]
)
...
...
@@ -169,6 +170,14 @@ class EnvironmentModel(QStandardItemModel):
version
=
self
.
data
(
version
)
return
{
"name"
:
name
,
"version"
:
version
}
def
environmentType
(
self
,
index
):
if
is_Qt_equal_or_higher
(
"5.11"
):
type_
=
index
.
siblingAtColumn
(
3
)
else
:
type_
=
index
.
sibling
(
index
.
row
(),
3
)
return
self
.
data
(
type_
)
class
ContainerWidget
(
ScrollWidget
):
"""Container widget to show block editors"""
...
...
@@ -235,6 +244,8 @@ class DatasetModel(QStringListModel):
class
ExperimentResources
:
"""Modelization of the experiments resources"""
def
__init__
(
self
,
context
=
None
):
self
.
context
=
context
...
...
@@ -256,20 +267,27 @@ class ExperimentResources:
self
.
refresh
()
def
refresh
(
self
):
if
self
.
context
is
None
:
return
ALGORITHM_TABLE_CLEANUP
=
"DROP TABLE IF EXISTS algorithms"
ALGORITHM_TABLE
=
"CREATE TABLE algorithms(name varchar, type varchar, inputs integer, outputs integer, is_analyzer boolean)"
INSERT_ALGORITHM
=
"INSERT INTO algorithms(name, type, inputs, outputs, is_analyzer) VALUES(?, ?, ?, ?, ?)"
QUEUE_TABLE_CLEANUP
=
"DROP TABLE IF EXISTS queues"
QUEUE_TABLE
=
"CREATE TABLE queues(name varchar, env_name varchar, env_version varchar, env_type varchar)"
INSERT_QUEUE
=
"INSERT INTO queues(name, env_name, env_version, env_type) VALUES (?, ?, ?, ?)"
query
=
QSqlQuery
()
if
not
query
.
exec_
(
ALGORITHM_TABLE_CLEANUP
):
raise
RuntimeError
(
f
"Failed to drop table:
{
query
.
lastError
().
text
()
}
"
)
for
query_str
in
[
ALGORITHM_TABLE_CLEANUP
,
QUEUE_TABLE_CLEANUP
]:
if
not
query
.
exec_
(
query_str
):
raise
RuntimeError
(
f
"Failed to drop table:
{
query
.
lastError
().
text
()
}
"
)
if
not
query
.
exec_
(
ALGORITHM_TABLE
):
raise
RuntimeError
(
f
"Failed to create table:
{
query
.
lastError
().
text
()
}
"
)
for
query_str
in
[
ALGORITHM_TABLE
,
QUEUE_TABLE
]:
if
not
query
.
exec_
(
query_str
):
raise
RuntimeError
(
f
"Failed to create table:
{
query
.
lastError
().
text
()
}
"
)
if
self
.
context
is
None
:
return
prefix_path
=
self
.
context
.
meta
[
"config"
].
path
model
=
AssetModel
()
...
...
@@ -304,6 +322,40 @@ class ExperimentResources:
f
"Failed to insert algorithm:
{
query
.
lastError
().
text
()
}
"
)
if
not
query
.
prepare
(
INSERT_QUEUE
):
raise
RuntimeError
(
f
"Failed to prepare query:
{
query
.
lastError
().
text
()
}
"
)
environments_path
=
self
.
context
.
meta
[
"environments"
]
with
open
(
environments_path
,
"rt"
)
as
file
:
environment_data
=
json
.
load
(
file
)
for
item
in
environment_data
.
get
(
"remote"
,
[]):
env_name
=
item
[
"name"
]
env_version
=
item
[
"version"
]
# import ipdb; ipdb.set_trace()
for
name
in
item
[
"queues"
].
keys
():
query
.
addBindValue
(
name
)
query
.
addBindValue
(
env_name
)
query
.
addBindValue
(
env_version
)
query
.
addBindValue
(
"remote"
)
if
not
query
.
exec_
():
raise
RuntimeError
(
f
"Failed to insert queue:
{
query
.
lastError
().
text
()
}
"
)
for
_
,
image_info
in
environment_data
.
get
(
"docker"
,
{}).
items
():
env_name
=
image_info
[
"name"
]
env_version
=
image_info
[
"version"
]
query
.
addBindValue
(
"Local"
)
query
.
addBindValue
(
env_name
)
query
.
addBindValue
(
env_version
)
query
.
addBindValue
(
"docker"
)
if
not
query
.
exec_
():
raise
RuntimeError
(
f
"Failed to insert queue:
{
query
.
lastError
().
text
()
}
"
)
class
AlgorithmResourceModel
(
QSqlTableModel
):
def
__init__
(
self
,
parent
=
None
):
...
...
@@ -361,6 +413,56 @@ class AlgorithmResourceModel(QSqlTableModel):
self
.
update
()
class
QueueResourceModel
(
QSqlTableModel
):
def
__init__
(
self
,
parent
=
None
):
super
().
__init__
(
parent
=
parent
)
self
.
_environment
=
None
self
.
_version
=
None
self
.
_type
=
None
self
.
setTable
(
"queues"
)
self
.
select
()
self
.
update
()
def
update
(
self
):
filter_str
=
""
if
self
.
_environment
is
not
None
:
filter_str
+=
f
"env_name='
{
self
.
_environment
}
'"
if
self
.
_version
is
not
None
:
if
filter_str
:
filter_str
+=
" AND "
filter_str
+=
f
"env_version='
{
self
.
_version
}
'"
if
self
.
_type
is
not
None
:
if
filter_str
:
filter_str
+=
" AND "
filter_str
+=
f
"env_type='
{
self
.
_type
}
'"
self
.
setFilter
(
filter_str
)
def
setEnvironment
(
self
,
name
,
version
):
if
self
.
_environment
==
name
and
self
.
_version
==
version
:
return
self
.
_environment
=
name
self
.
_version
=
version
self
.
update
()
def
setType
(
self
,
type_
):
if
self
.
_type
==
type_
:
return
self
.
_type
=
type_
self
.
update
()
def
dump
(
self
):
print
(
self
.
filter
())
for
i
in
range
(
self
.
rowCount
()):
print
([
self
.
index
(
i
,
j
).
data
()
for
j
in
range
(
4
)])
# ------------------------------------------------------------------------------
# Editors
...
...
@@ -683,11 +785,13 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
self
.
_queue_enabled
=
True
self
.
parameter_item
=
None
self
.
algorithm_model
=
None
self
.
queue_model
=
QueueResourceModel
()
self
.
algorithm_combobox
=
QComboBox
()
self
.
algorithm_combobox
.
setObjectName
(
"algorithms"
)
self
.
environment_combobox
=
QComboBox
()
self
.
environment_combobox
.
setObjectName
(
"environments"
)
self
.
queue_combobox
=
QComboBox
()
self
.
queue_combobox
.
setModel
(
self
.
queue_model
)
self
.
parameters_editor
=
AlgorithmParametersEditor
(
prefix_path
)
self
.
remap_button
=
QPushButton
(
QIcon
(
":/resources/remap"
),
""
)
...
...
@@ -724,6 +828,9 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
self
.
environment_combobox
.
currentIndexChanged
.
connect
(
lambda
*
args
:
setattr
(
self
,
"environment_changed"
,
True
)
)
self
.
environment_combobox
.
currentIndexChanged
.
connect
(
self
.
__onEnvironmentChanged
)
self
.
queue_combobox
.
currentIndexChanged
.
connect
(
self
.
dataChanged
)
self
.
queue_combobox
.
currentIndexChanged
.
connect
(
lambda
*
args
:
setattr
(
self
,
"queue_changed"
,
True
)
...
...
@@ -765,6 +872,15 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
self
.
_queue_enabled
)
@
pyqtSlot
(
int
)
def
__onEnvironmentChanged
(
self
,
row
):
model
=
self
.
environment_combobox
.
model
()
index
=
model
.
index
(
row
,
0
)
self
.
queue_model
.
setEnvironment
(
**
model
.
environment
(
index
))
self
.
queue_model
.
setType
(
model
.
environmentType
(
index
))
self
.
queue_combobox
.
setCurrentIndex
(
0
)
@
pyqtSlot
(
str
)
def
__onAlgorithmChanged
(
self
,
algorithm_name
):
...
...
@@ -814,9 +930,6 @@ class ExecutionPropertiesEditor(AbstractBaseEditor):
def
environmentModel
(
self
):
return
self
.
environment_combobox
.
model
()
def
setQueueModel
(
self
,
model
):
self
.
queue_combobox
.
setModel
(
model
)
def
setQueueEnabled
(
self
,
enabled
):
self
.
_queue_enabled
=
enabled
self
.
__updateUi
()
...
...
@@ -905,11 +1018,6 @@ class AbstractBlockEditor(AbstractBaseEditor):
raise
NotImplementedError
def
setQueueModel
(
self
,
model
):
"""Setup queue model"""
raise
NotImplementedError
class
BlockEditor
(
AbstractBlockEditor
):
def
__init__
(
self
,
block_name
,
prefix_path
,
parent
=
None
):
...
...
@@ -942,9 +1050,6 @@ class BlockEditor(AbstractBlockEditor):
def
environmentModel
(
self
):
return
self
.
properties_editor
.
environmentModel
()
def
setQueueModel
(
self
,
model
):
self
.
properties_editor
.
setQueueModel
(
model
)
def
load
(
self
,
json_object
):
self
.
properties_editor
.
load
(
json_object
)
...
...
@@ -1019,9 +1124,6 @@ class LoopBlockEditor(AbstractBlockEditor):
def
environmentModel
(
self
):
return
self
.
processor_properties_editor
.
environmentModel
()
def
setQueueModel
(
self
,
model
):
self
.
processor_properties_editor
.
setQueueModel
(
model
)
def
load
(
self
,
json_object
):
processor_data
=
{
key
[
len
(
PROCESSOR_PREFIX
)
:]:
value
...
...
@@ -1087,9 +1189,11 @@ class GlobalParametersEditor(AbstractBaseEditor):
self
.
json_object
=
{}
self
.
environment_changed
=
False
self
.
queue_changed
=
False
self
.
queue_model
=
QueueResourceModel
()
self
.
environment_combobox
=
QComboBox
()
self
.
queue_combobox
=
QComboBox
()
self
.
queue_combobox
.
setModel
(
self
.
queue_model
)
self
.
parameters_editor_listwidget
=
EditorListWidget
()
form_layout
=
QFormLayout
()
...
...
@@ -1105,18 +1209,29 @@ class GlobalParametersEditor(AbstractBaseEditor):
self
.
environment_combobox
.
currentIndexChanged
.
connect
(
lambda
*
args
:
setattr
(
self
,
"environment_changed"
,
True
)
)
self
.
environment_combobox
.
currentIndexChanged
.
connect
(
self
.
__onEnvironmentChanged
)
self
.
queue_combobox
.
currentIndexChanged
.
connect
(
self
.
dataChanged
)
self
.
queue_combobox
.
currentIndexChanged
.
connect
(
lambda
*
args
:
setattr
(
self
,
"queue_changed"
,
True
)
)
self
.
parameters_editor_listwidget
.
dataChanged
.
connect
(
self
.
dataChanged
)
self
.
prefixPathChanged
.
connect
(
self
.
queue_model
.
select
)
@
pyqtSlot
(
int
)
def
__onEnvironmentChanged
(
self
,
row
):
model
=
self
.
environment_combobox
.
model
()
index
=
model
.
index
(
row
,
0
)
self
.
queue_model
.
setEnvironment
(
**
model
.
environment
(
index
))
self
.
queue_model
.
setType
(
model
.
environmentType
(
index
))
self
.
queue_combobox
.
setCurrentIndex
(
0
)
def
setEnvironmentModel
(
self
,
model
):
self
.
environment_combobox
.
setModel
(
model
)
def
setQueueModel
(
self
,
model
):
self
.
queue_combobox
.
setModel
(
model
)
def
clear
(
self
):
self
.
environment_combobox
.
setCurrentIndex
(
0
)
self
.
parameters_editor_listwidget
.
clear
()
...
...
@@ -1204,11 +1319,9 @@ class ExperimentEditor(AbstractAssetEditor):
self
.
setObjectName
(
self
.
__class__
.
__name__
)
self
.
set_title
(
self
.
tr
(
"Experiment"
))
self
.
p
re
fix
_model
=
ExperimentResources
()
self
.
re
source
_model
=
ExperimentResources
()
self
.
processing_env_model
=
EnvironmentModel
()
self
.
queue_model
=
QStringListModel
()
self
.
queue_model
.
setStringList
([
"Local"
])
self
.
dataset_model
=
DatasetModel
()
...
...
@@ -1222,7 +1335,6 @@ class ExperimentEditor(AbstractAssetEditor):
self
.
analyzers_widget
=
ContainerWidget
()
self
.
globalparameters_widget
=
GlobalParametersEditor
(
self
.
prefixPath
())
self
.
globalparameters_widget
.
setEnvironmentModel
(
self
.
processing_env_model
)
self
.
globalparameters_widget
.
setQueueModel
(
self
.
queue_model
)
self
.
tabwidget
=
QTabWidget
()
self
.
tabwidget
.
addTab
(
self
.
datasets_widget
,
self
.
tr
(
"Datasets"
))
...
...
@@ -1250,7 +1362,7 @@ class ExperimentEditor(AbstractAssetEditor):
@
pyqtSlot
()
def
__update
(
self
):
self
.
p
re
fix
_model
.
setContext
(
self
.
context
)
self
.
re
source
_model
.
setContext
(
self
.
context
)
for
object_
in
[
self
.
algorithm_model
,
...
...
@@ -1396,7 +1508,7 @@ class ExperimentEditor(AbstractAssetEditor):
def
_load_json
(
self
,
json_object
):
"""Load the json object passed as parameter"""
self
.
p
re
fix
_model
.
refresh
()
self
.
re
source
_model
.
refresh
()
for
widget
in
[
self
.
datasets_widget
,
...
...
@@ -1427,7 +1539,6 @@ class ExperimentEditor(AbstractAssetEditor):
for
name
,
data
in
items
.
items
():
editor
=
editor_klass
(
name
,
self
.
prefix_path
)
editor
.
setEnvironmentModel
(
self
.
processing_env_model
)
editor
.
setQueueModel
(
self
.
queue_model
)
editor
.
load
(
data
)
editor
.
algorithmChanged
.
connect
(
self
.
__onAlgorithmChanged
)
container
.
addWidget
(
editor
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment