Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
bob.fusion.base
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
bob
bob.fusion.base
Commits
b68afbd1
Commit
b68afbd1
authored
8 years ago
by
Amir Mohammadi
Browse files
Options
Downloads
Patches
Plain Diff
refactor fusion script, fix bugs
parent
f643a2c2
Branches
Branches containing commit
Tags
v1.2.1
Tags containing commit
1 merge request
!2
First Alpha release
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
bob/fusion/base/script/bob_fuse.py
+152
-121
152 additions, 121 deletions
bob/fusion/base/script/bob_fuse.py
with
152 additions
and
121 deletions
bob/fusion/base/script/bob_fuse.py
+
152
−
121
View file @
b68afbd1
...
...
@@ -18,6 +18,73 @@ import bob.core
logger
=
bob
.
core
.
log
.
setup
(
"
bob.fusion.base
"
)
def
save_fused_scores
(
save_path
,
fused_scores
,
score_lines
):
score_lines
[
'
score
'
]
=
fused_scores
gen
,
zei
,
atk
,
_
,
_
,
_
=
get_2negatives_1positive
(
score_lines
)
create_directories_safe
(
os
.
path
.
dirname
(
save_path
))
dump_score
(
save_path
,
score_lines
)
dump_score
(
save_path
+
'
-licit
'
,
np
.
append
(
gen
,
zei
))
dump_score
(
save_path
+
'
-spoof
'
,
np
.
append
(
gen
,
atk
))
def
routine_fusion
(
algorithm
,
model_file
,
scores_train_lines
,
scores_train
,
train_neg
,
train_pos
,
fused_train_file
,
scores_dev_lines
=
None
,
scores_dev
=
None
,
dev_neg
=
None
,
dev_pos
=
None
,
fused_dev_file
=
None
,
scores_eval_lines
=
None
,
scores_eval
=
None
,
fused_eval_file
=
None
,
force
=
False
,
min_file_size
=
1000
,
):
# train the preprocessors
train_scores
=
np
.
vstack
((
train_neg
,
train_pos
))
neg_len
=
train_neg
.
shape
[
0
]
y
=
np
.
zeros
((
train_scores
.
shape
[
0
],),
dtype
=
'
bool
'
)
y
[
neg_len
:]
=
True
algorithm
.
train_preprocessors
(
train_scores
,
y
)
# preprocess data
scores_train
=
algorithm
.
preprocess
(
scores_train
)
train_neg
,
train_pos
=
algorithm
.
preprocess
(
train_neg
),
algorithm
.
preprocess
(
train_pos
)
if
scores_dev
is
not
None
:
scores_dev
=
algorithm
.
preprocess
(
scores_dev
)
dev_neg
,
dev_pos
=
algorithm
.
preprocess
(
dev_neg
),
algorithm
.
preprocess
(
dev_pos
)
if
scores_eval
is
not
None
:
scores_eval
=
algorithm
.
preprocess
(
scores_eval
)
# train the model
if
utils
.
check_file
(
model_file
,
force
,
min_file_size
):
logger
.
info
(
"
model
'
%s
'
already exists.
"
,
model_file
)
algorithm
=
algorithm
.
load
(
model_file
)
else
:
algorithm
.
train
(
train_neg
,
train_pos
,
dev_neg
,
dev_pos
)
algorithm
.
save
(
model_file
)
# fuse the scores (train)
if
utils
.
check_file
(
fused_train_file
,
force
,
min_file_size
):
logger
.
info
(
"
score file
'
%s
'
already exists.
"
,
fused_train_file
)
else
:
fused_scores_train
=
algorithm
.
fuse
(
scores_train
)
save_fused_scores
(
fused_train_file
,
fused_scores_train
,
scores_train_lines
)
if
scores_dev
is
not
None
:
# fuse the scores (dev)
if
utils
.
check_file
(
fused_dev_file
,
force
,
min_file_size
):
logger
.
info
(
"
score file
'
%s
'
already exists.
"
,
fused_dev_file
)
else
:
fused_scores_dev
=
algorithm
.
fuse
(
scores_dev
)
save_fused_scores
(
fused_dev_file
,
fused_scores_dev
,
scores_dev_lines
)
if
scores_eval
is
not
None
:
# fuse the scores (eval)
if
utils
.
check_file
(
fused_eval_file
,
force
,
min_file_size
):
logger
.
info
(
"
score file
'
%s
'
already exists.
"
,
fused_eval_file
)
else
:
fused_scores_eval
=
algorithm
.
fuse
(
scores_eval
)
save_fused_scores
(
fused_eval_file
,
fused_scores_eval
,
scores_eval_lines
)
def
fuse
(
args
,
command_line_parameters
):
"""
Do the actual fusion.
"""
algorithm
=
args
.
algorithm
...
...
@@ -54,6 +121,7 @@ def fuse(args, command_line_parameters):
check_consistency
(
gen_le
,
zei_le
,
atk_le
)
scores_train
=
get_scores
(
gen_lt
,
zei_lt
,
atk_lt
)
scores_train_lines
=
get_score_lines
(
gen_lt
[
0
:
1
],
zei_lt
[
0
:
1
],
atk_lt
[
0
:
1
])
train_neg
=
get_scores
(
zei_lt
,
atk_lt
)
train_pos
=
get_scores
(
gen_lt
)
if
args
.
dev_files
:
...
...
@@ -62,74 +130,36 @@ def fuse(args, command_line_parameters):
dev_neg
=
get_scores
(
zei_ld
,
atk_ld
)
dev_pos
=
get_scores
(
gen_ld
)
else
:
dev_neg
,
dev_pos
=
None
,
None
scores_dev
,
scores_dev_lines
,
dev_neg
,
dev_pos
=
None
,
None
,
None
,
None
if
args
.
eval_files
:
scores_eval
=
get_scores
(
gen_le
,
zei_le
,
atk_le
)
scores_eval_lines
=
get_score_lines
(
gen_le
[
0
:
1
],
zei_le
[
0
:
1
],
atk_le
[
0
:
1
])
else
:
scores_eval
,
scores_eval_lines
=
None
,
None
# check for nan values
found_nan
=
False
found_nan
,
_
,
scores_train
=
remove_nan
(
scores_train
,
found_nan
)
found_nan
,
nan_train
,
scores_train
=
remove_nan
(
scores_train
,
found_nan
)
scores_train_lines
=
scores_train_lines
[
~
nan_train
]
found_nan
,
_
,
train_neg
=
remove_nan
(
train_neg
,
found_nan
)
found_nan
,
_
,
train_pos
=
remove_nan
(
train_pos
,
found_nan
)
if
args
.
dev_files
:
found_nan
,
nan_dev
,
scores_dev
=
remove_nan
(
scores_dev
,
found_nan
)
scores_dev_lines
=
scores_dev_lines
[
~
nan_dev
]
found_nan
,
_
,
dev_neg
=
remove_nan
(
dev_neg
,
found_nan
)
found_nan
,
_
,
dev_pos
=
remove_nan
(
dev_pos
,
found_nan
)
if
args
.
eval_files
:
found_nan
,
nan_eval
,
scores_eval
=
remove_nan
(
scores_eval
,
found_nan
)
scores_eval_lines
=
scores_eval_lines
[
~
nan_eval
]
if
found_nan
:
logger
.
warn
(
'
Some nan values were removed.
'
)
# train the preprocessors
algorithm
.
train_preprocessors
(
scores_train
)
# preprocess data
train_neg
,
train_pos
=
algorithm
.
preprocess
(
train_neg
),
algorithm
.
preprocess
(
train_pos
)
if
args
.
dev_files
:
scores_dev
=
algorithm
.
preprocess
(
scores_dev
)
dev_neg
,
dev_pos
=
algorithm
.
preprocess
(
dev_neg
),
algorithm
.
preprocess
(
dev_pos
)
if
args
.
eval_files
:
scores_eval
=
algorithm
.
preprocess
(
scores_eval
)
# train the model
if
utils
.
check_file
(
args
.
model_file
,
args
.
force
,
1000
):
logger
.
info
(
"
- Fusion: model
'
%s
'
already exists.
"
,
args
.
model_file
)
algorithm
=
algorithm
.
load
(
args
.
model_file
)
else
:
algorithm
.
train
(
train_neg
,
train_pos
,
dev_neg
,
dev_pos
)
algorithm
.
save
(
args
.
model_file
)
# fuse the scores (dev)
if
utils
.
check_file
(
args
.
fused_dev_file
,
args
.
force
,
1000
):
logger
.
info
(
"
- Fusion: scores
'
%s
'
already exists.
"
,
args
.
fused_dev_file
)
elif
args
.
dev_files
:
fused_scores_dev
=
algorithm
.
fuse
(
scores_dev
)
score_lines
=
scores_dev_lines
[
~
nan_dev
]
score_lines
[
'
score
'
]
=
fused_scores_dev
gen
,
zei
,
atk
,
_
,
_
,
_
=
get_2negatives_1positive
(
score_lines
)
create_directories_safe
(
os
.
path
.
dirname
(
args
.
fused_dev_file
))
dump_score
(
args
.
fused_dev_file
,
score_lines
)
dump_score
(
args
.
fused_dev_file
+
'
-licit
'
,
np
.
append
(
gen
,
zei
))
dump_score
(
args
.
fused_dev_file
+
'
-spoof
'
,
np
.
append
(
gen
,
atk
))
# fuse the scores (eval)
if
args
.
eval_files
:
if
utils
.
check_file
(
args
.
fused_eval_file
,
args
.
force
,
1000
):
logger
.
info
(
"
- Fusion: scores
'
%s
'
already exists.
"
,
args
.
fused_eval_file
)
else
:
fused_scores_eval
=
algorithm
.
fuse
(
scores_eval
)
score_lines
=
scores_eval_lines
[
~
nan_eval
]
score_lines
[
'
score
'
]
=
fused_scores_eval
gen
,
zei
,
atk
,
_
,
_
,
_
=
get_2negatives_1positive
(
score_lines
)
create_directories_safe
(
os
.
path
.
dirname
(
args
.
fused_eval_file
))
dump_score
(
args
.
fused_eval_file
,
score_lines
)
dump_score
(
args
.
fused_eval_file
+
'
-licit
'
,
np
.
append
(
gen
,
zei
))
dump_score
(
args
.
fused_eval_file
+
'
-spoof
'
,
np
.
append
(
gen
,
atk
))
routine_fusion
(
algorithm
,
args
.
model_file
,
scores_train_lines
,
scores_train
,
train_neg
,
train_pos
,
args
.
fused_train_file
,
scores_dev_lines
,
scores_dev
,
dev_neg
,
dev_pos
,
args
.
fused_dev_file
,
scores_eval_lines
,
scores_eval
,
args
.
fused_eval_file
,
args
.
force
)
def
main
(
command_line_parameters
=
None
):
...
...
@@ -145,5 +175,6 @@ def main(command_line_parameters=None):
logger
.
error
(
"
During the execution, an exception was raised: %s
"
%
e
)
raise
if
__name__
==
"
__main__
"
:
main
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment