Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
bob.measure
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
2
Issues
2
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
bob
bob.measure
Commits
8d99cfaf
Commit
8d99cfaf
authored
Oct 22, 2016
by
Amir Mohammadi
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Load score more efficiently for negatives and positives
parent
c3c1ecbc
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
25 additions
and
3 deletions
+25
-3
bob/measure/load.py
bob/measure/load.py
+19
-3
bob/measure/test_io.py
bob/measure/test_io.py
+6
-0
No files found.
bob/measure/load.py
View file @
8d99cfaf
...
...
@@ -264,8 +264,7 @@ def cmc_five_column(filename):
return
_split_cmc_scores
(
score_lines
,
2
)
def
load_score
(
filename
,
ncolumns
=
None
):
def
load_score
(
filename
,
ncolumns
=
None
,
minimal
=
False
,
**
kwargs
):
"""Load scores using numpy.loadtxt and return the data as a numpy array.
Parameters:
...
...
@@ -277,6 +276,11 @@ def load_score(filename, ncolumns=None):
specifying the number of columns in the score file. If None is provided,
the number of columns will be guessed.
minimal (:py:class:`bool`, optional): If True, only loads ``claimed_id``, ``real_id``,
and ``scores``.
**kwargs: Keyword arguments passed to :py:func:`numpy.genfromtxt`
Returns:
...
...
@@ -300,6 +304,7 @@ def load_score(filename, ncolumns=None):
finally
:
f
.
close
()
usecols
=
kwargs
.
pop
(
'usecols'
,
None
)
if
ncolumns
==
4
:
names
=
(
'claimed_id'
,
'real_id'
,
'test_label'
,
'score'
)
converters
=
{
...
...
@@ -307,6 +312,8 @@ def load_score(filename, ncolumns=None):
1
:
convertfunc
,
2
:
convertfunc
,
3
:
float
}
if
minimal
:
usecols
=
(
0
,
1
,
3
)
elif
ncolumns
==
5
:
names
=
(
'claimed_id'
,
'model_label'
,
'real_id'
,
'test_label'
,
'score'
)
...
...
@@ -316,12 +323,14 @@ def load_score(filename, ncolumns=None):
2
:
convertfunc
,
3
:
convertfunc
,
4
:
float
}
if
minimal
:
usecols
=
(
0
,
2
,
4
)
else
:
raise
ValueError
(
"ncolumns of 4 and 5 are supported only."
)
score_lines
=
numpy
.
genfromtxt
(
open_file
(
filename
,
mode
=
'rb'
),
dtype
=
None
,
names
=
names
,
converters
=
converters
,
invalid_raise
=
True
)
converters
=
converters
,
invalid_raise
=
True
,
usecols
=
usecols
,
**
kwargs
)
new_dtype
=
[]
for
name
in
score_lines
.
dtype
.
names
[:
-
1
]:
new_dtype
.
append
((
name
,
str
(
score_lines
.
dtype
[
name
]).
replace
(
'S'
,
'U'
)))
...
...
@@ -342,6 +351,13 @@ def get_negatives_positives(score_lines):
return
(
negatives
,
positives
)
def
get_negatives_positives_from_file
(
filename
,
**
kwargs
):
"""Loads the scores first efficiently and then calls
get_negatives_positives"""
score_lines
=
load_score
(
filename
,
minimal
=
True
,
**
kwargs
)
return
get_negatives_positives
(
score_lines
)
def
get_negatives_positives_all
(
score_lines_list
):
"""Take a list of outputs of load_score and return stacked negatives and
positives.
...
...
bob/measure/test_io.py
View file @
8d99cfaf
...
...
@@ -60,6 +60,12 @@ def test_load_score():
for
name
in
normal_scores
.
dtype
.
names
:
assert
all
(
normal_scores
[
name
]
==
compressed_scores
[
name
])
# test minimal loading
minimal_scores
=
bob
.
measure
.
load
.
load_score
(
normal_score_file
,
minimal
=
True
)
assert
len
(
minimal_scores
)
==
910
assert
len
(
minimal_scores
.
dtype
)
==
3
assert
minimal_scores
.
dtype
.
names
==
(
'claimed_id'
,
'real_id'
,
'score'
)
def
test_dump_score
():
# This function tests the IO functionality of dumping score files
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment