Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
beat.web
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
beat
beat.web
Commits
2acd2cf5
"README.md" did not exist on "fdece21fb429ed11513cd6ae3b4138fd56ad5765"
Commit
2acd2cf5
authored
5 years ago
by
Samuel GAIST
Browse files
Options
Downloads
Patches
Plain Diff
[libraries][models] Pre-commit cleanup
parent
9ed89f38
No related branches found
No related tags found
1 merge request
!324
Return compatibility information for algorithms and libraries
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
beat/web/libraries/models.py
+86
-69
86 additions, 69 deletions
beat/web/libraries/models.py
with
86 additions
and
69 deletions
beat/web/libraries/models.py
+
86
−
69
View file @
2acd2cf5
...
@@ -43,18 +43,20 @@ import simplejson
...
@@ -43,18 +43,20 @@ import simplejson
import
collections
import
collections
#----------------------------------------------------------
#
----------------------------------------------------------
def
validate_library
(
declaration
):
def
validate_library
(
declaration
):
"""
Validates the declaration of a library code, returns wrapper
"""
"""
Validates the declaration of a library code, returns wrapper
"""
if
not
declaration
:
if
not
declaration
:
raise
SyntaxError
(
'
Library declaration cannot be empty
'
)
raise
SyntaxError
(
"
Library declaration cannot be empty
"
)
if
not
(
isinstance
(
declaration
,
dict
)):
if
not
(
isinstance
(
declaration
,
dict
)):
try
:
try
:
declaration_dict
=
simplejson
.
loads
(
declaration
,
object_pairs_hook
=
collections
.
OrderedDict
)
declaration_dict
=
simplejson
.
loads
(
declaration
,
object_pairs_hook
=
collections
.
OrderedDict
)
except
Exception
as
e
:
except
Exception
as
e
:
raise
SyntaxError
(
str
(
e
))
raise
SyntaxError
(
str
(
e
))
else
:
else
:
...
@@ -63,111 +65,122 @@ def validate_library(declaration):
...
@@ -63,111 +65,122 @@ def validate_library(declaration):
library
=
beat
.
core
.
library
.
Library
(
settings
.
PREFIX
,
declaration_dict
)
library
=
beat
.
core
.
library
.
Library
(
settings
.
PREFIX
,
declaration_dict
)
if
not
library
.
valid
:
if
not
library
.
valid
:
raise
SyntaxError
(
'
\n
* %s
'
%
'
\n
*
'
.
join
(
library
.
errors
))
raise
SyntaxError
(
"
\n
* %s
"
%
"
\n
*
"
.
join
(
library
.
errors
))
return
library
return
library
#----------------------------------------------------------
#
----------------------------------------------------------
class
LibraryStorage
(
OverwriteStorage
):
class
LibraryStorage
(
OverwriteStorage
):
def
__init__
(
self
,
*
args
,
**
kwargs
):
def
__init__
(
self
,
*
args
,
**
kwargs
):
kwargs
[
'
location
'
]
=
settings
.
LIBRARIES_ROOT
kwargs
[
"
location
"
]
=
settings
.
LIBRARIES_ROOT
super
(
LibraryStorage
,
self
).
__init__
(
*
args
,
**
kwargs
)
super
(
LibraryStorage
,
self
).
__init__
(
*
args
,
**
kwargs
)
#----------------------------------------------------------
#
----------------------------------------------------------
class
LibraryManager
(
CodeManager
):
class
LibraryManager
(
CodeManager
):
def
create_library
(
def
create_library
(
self
,
author
,
name
,
short_description
=
''
,
self
,
description
=
''
,
declaration
=
None
,
code
=
None
,
version
=
1
,
author
,
previous_version
=
None
,
fork_of
=
None
):
name
,
short_description
=
""
,
description
=
""
,
declaration
=
None
,
code
=
None
,
version
=
1
,
previous_version
=
None
,
fork_of
=
None
,
):
default
=
beat
.
core
.
library
.
Library
(
settings
.
PREFIX
,
data
=
None
)
default
=
beat
.
core
.
library
.
Library
(
settings
.
PREFIX
,
data
=
None
)
return
self
.
create_code
(
author
,
name
,
default
,
short_description
,
return
self
.
create_code
(
description
,
declaration
,
code
,
version
,
author
,
previous_version
,
fork_of
)
name
,
default
,
short_description
,
description
,
declaration
,
code
,
version
,
previous_version
,
fork_of
,
)
#----------------------------------------------------------
#
----------------------------------------------------------
class
Library
(
Code
):
class
Library
(
Code
):
#_____ Fields __________
#
_____ Fields __________
declaration_file
=
models
.
FileField
(
declaration_file
=
models
.
FileField
(
storage
=
LibraryStorage
(),
storage
=
LibraryStorage
(),
upload_to
=
get_contribution_declaration_filename
,
upload_to
=
get_contribution_declaration_filename
,
blank
=
True
,
null
=
True
,
blank
=
True
,
null
=
True
,
max_length
=
200
,
max_length
=
200
,
db_column
=
'
declaration
'
db_column
=
"
declaration
"
,
)
)
description_file
=
models
.
FileField
(
description_file
=
models
.
FileField
(
storage
=
LibraryStorage
(),
storage
=
LibraryStorage
(),
upload_to
=
get_contribution_description_filename
,
upload_to
=
get_contribution_description_filename
,
blank
=
True
,
null
=
True
,
blank
=
True
,
null
=
True
,
max_length
=
200
,
max_length
=
200
,
db_column
=
'
description
'
db_column
=
"
description
"
,
)
)
source_code_file
=
models
.
FileField
(
source_code_file
=
models
.
FileField
(
storage
=
LibraryStorage
(),
storage
=
LibraryStorage
(),
upload_to
=
get_contribution_source_code_filename
,
upload_to
=
get_contribution_source_code_filename
,
blank
=
True
,
null
=
True
,
blank
=
True
,
null
=
True
,
max_length
=
200
,
max_length
=
200
,
db_column
=
'
source_code
'
db_column
=
"
source_code
"
,
)
)
# Read-only parameters that are updated at every save(), if required
# Read-only parameters that are updated at every save(), if required
referenced_libraries
=
models
.
ManyToManyField
(
'
self
'
,
referenced_libraries
=
models
.
ManyToManyField
(
blank
=
True
,
related_name
=
'
referencing
'
,
"
self
"
,
blank
=
True
,
related_name
=
"
referencing
"
,
symmetrical
=
False
symmetrical
=
False
)
)
objects
=
LibraryManager
()
objects
=
LibraryManager
()
# _____ Meta parameters __________
#_____ Meta parameters __________
class
Meta
(
Code
.
Meta
):
class
Meta
(
Code
.
Meta
):
verbose_name_plural
=
'
libraries
'
verbose_name_plural
=
"
libraries
"
# _____ Utilities __________
#_____ Utilities __________
def
get_absolute_url
(
self
):
def
get_absolute_url
(
self
):
return
reverse
(
return
reverse
(
'
libraries:view
'
,
"
libraries:view
"
,
args
=
(
self
.
author
.
username
,
self
.
name
,
self
.
version
)
args
=
(
self
.
author
.
username
,
self
.
name
,
self
.
version
,),
)
)
def
get_api_update_url
(
self
):
def
get_api_update_url
(
self
):
'''
Returns the endpoint to update this object
'''
"""
Returns the endpoint to update this object
"""
return
reverse
(
return
reverse
(
'
api_libraries:object
'
,
"
api_libraries:object
"
,
args
=
(
self
.
author
.
username
,
self
.
name
,
self
.
version
)
args
=
(
self
.
author
.
username
,
self
.
name
,
self
.
version
,),
)
)
def
get_api_share_url
(
self
):
def
get_api_share_url
(
self
):
'''
Returns the endpoint to share this object
'''
"""
Returns the endpoint to share this object
"""
return
reverse
(
return
reverse
(
'
api_libraries:share
'
,
"
api_libraries:share
"
,
args
=
(
self
.
author
.
username
,
self
.
name
,
self
.
version
)
args
=
(
self
.
author
.
username
,
self
.
name
,
self
.
version
,),
)
)
# _____ Overrides __________
#_____ Overrides __________
def
save
(
self
,
*
args
,
**
kwargs
):
def
save
(
self
,
*
args
,
**
kwargs
):
wrapper
=
self
.
_save_preprocessing
()
wrapper
=
self
.
_save_preprocessing
()
...
@@ -179,50 +192,47 @@ class Library(Code):
...
@@ -179,50 +192,47 @@ class Library(Code):
if
wrapper
.
uses
is
not
None
:
if
wrapper
.
uses
is
not
None
:
for
l
in
set
(
wrapper
.
uses
.
values
()):
for
l
in
set
(
wrapper
.
uses
.
values
()):
s
=
beat
.
core
.
library
.
Storage
(
settings
.
PREFIX
,
l
)
s
=
beat
.
core
.
library
.
Storage
(
settings
.
PREFIX
,
l
)
library
=
Library
.
objects
.
get
(
author__username
=
s
.
username
,
library
=
Library
.
objects
.
get
(
name
=
s
.
name
,
author__username
=
s
.
username
,
name
=
s
.
name
,
version
=
s
.
version
version
=
s
.
version
,
)
)
self
.
referenced_libraries
.
add
(
library
)
self
.
referenced_libraries
.
add
(
library
)
# _____ Methods __________
#_____ Methods __________
def
validate
(
self
,
declaration
):
def
validate
(
self
,
declaration
):
return
validate_library
(
declaration
)
return
validate_library
(
declaration
)
def
modifiable
(
self
):
def
modifiable
(
self
):
"""
Can modify if nobody points at me
"""
"""
Can modify if nobody points at me
"""
return
super
(
Library
,
self
).
modifiable
()
and
((
self
.
referencing
.
count
()
+
self
.
used_by_algorithms
.
count
())
==
0
)
return
super
(
Library
,
self
).
modifiable
()
and
(
(
self
.
referencing
.
count
()
+
self
.
used_by_algorithms
.
count
())
==
0
)
def
deletable
(
self
):
def
deletable
(
self
):
"""
Can delete if nobody points at me
"""
"""
Can delete if nobody points at me
"""
return
super
(
Library
,
self
).
deletable
()
and
((
self
.
referencing
.
count
()
+
self
.
used_by_algorithms
.
count
())
==
0
)
return
super
(
Library
,
self
).
deletable
()
and
(
(
self
.
referencing
.
count
()
+
self
.
used_by_algorithms
.
count
())
==
0
)
def
valid
(
self
):
def
valid
(
self
):
return
True
# A library (at least for now) is always implemented in Python,
return
True
# A library (at least for now) is always implemented in Python,
# thus always valid
# thus always valid
def
core
(
self
):
def
core
(
self
):
return
validate_library
(
self
.
declaration
)
return
validate_library
(
self
.
declaration
)
def
uses
(
self
):
def
uses
(
self
):
return
self
.
core
().
uses
return
self
.
core
().
uses
def
environments
(
self
):
def
environments
(
self
):
'''
Calculates environment usage for this library
"""
Calculates environment usage for this library
Returns:
Returns:
list: mapping environment to usage counts, determining how many times
list: mapping environment to usage counts, determining how many times
a given algorithm has been successfuly used on that environment
a given algorithm has been successful
l
y used on that environment
'''
"""
from
django.db.models
import
Count
,
Q
from
django.db.models
import
Count
,
Q
from
..experiments.models
import
Block
from
..experiments.models
import
Block
...
@@ -230,18 +240,25 @@ class Library(Code):
...
@@ -230,18 +240,25 @@ class Library(Code):
from
..backend.models
import
Environment
from
..backend.models
import
Environment
# Tries to figure through a maximum if using algorithms have been
# Tries to figure through a maximum if using algorithms have been
# successfuly used inside an environment.
# successful
l
y used inside an environment.
# Case 1) The block is part of an experiment that was successful
# Case 1) The block is part of an experiment that was successful
# Case 2) The block is part of an experiment that is not successful
# Case 2) The block is part of an experiment that is not successful
# (failed or other), but it is CACHED (if not cached, then we can't
# (failed or other), but it is CACHED (if not cached, then we can't
# attest anything about the algorithm/environment relationship!)
# attest anything about the algorithm/environment relationship!)
envs
=
Environment
.
objects
.
filter
(
blocks__in
=
Block
.
objects
.
filter
(
\
envs
=
(
algorithm__in
=
self
.
used_by_algorithms
.
all
()).
filter
(
\
Environment
.
objects
.
filter
(
Q
(
experiment__status
=
Experiment
.
DONE
)
|
\
blocks__in
=
Block
.
objects
.
filter
(
((
~
Q
(
experiment__status
=
Experiment
.
DONE
))
&
Q
(
status
=
Block
.
DONE
))
algorithm__in
=
self
.
used_by_algorithms
.
all
()
)).
annotate
(
itemcount
=
Count
(
'
id
'
)).
order_by
(
'
-creation_date
'
\
).
filter
(
).
distinct
()
Q
(
experiment__status
=
Experiment
.
DONE
)
|
((
~
Q
(
experiment__status
=
Experiment
.
DONE
))
&
Q
(
status
=
Block
.
DONE
))
)
)
.
annotate
(
itemcount
=
Count
(
"
id
"
))
.
order_by
(
"
-creation_date
"
)
.
distinct
()
)
return
[(
k
,
k
.
itemcount
)
for
k
in
envs
]
return
[(
k
,
k
.
itemcount
)
for
k
in
envs
]
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment