Skip to content
Snippets Groups Projects
Commit 18ecfb37 authored by Manuel Günther's avatar Manuel Günther
Browse files

Moved loading of scores into separate function

parent 504a69fc
No related branches found
No related tags found
1 merge request!20Resolve "load_scores extremely memory hungry"
Pipeline #
......@@ -93,16 +93,8 @@ def four_column(filename):
float: The result of the comparison of the model and the probe
"""
return _iterate_score_file(filename)
opened = open_file(filename, 'rb')
if sys.version_info.major > 2:
import io
opened = io.TextIOWrapper(opened, newline="")
reader = csv.reader(opened, delimiter=' ')
for splits in reader:
splits[-1] = float(splits[-1])
yield splits
def split_four_column(filename):
......@@ -220,15 +212,7 @@ def five_column(filename):
"""
opened = open_file(filename, 'rb')
if sys.version_info.major > 2:
import io
opened = io.TextIOWrapper(opened, newline="")
reader = csv.reader(opened, delimiter=' ')
for splits in reader:
splits[-1] = float(splits[-1])
yield splits
return _iterate_score_file(filename)
def split_five_column(filename):
......@@ -421,6 +405,22 @@ def dump_score(filename, score_lines):
numpy.savetxt(filename, score_lines, fmt=fmt)
def _iterate_score_file(filename):
"""Opens the score file for reading and yields the score file line by line in a tuple/list.
The last element of the line (which is the score) will be transformed to float, the other elements will be str
"""
opened = open_file(filename, 'rb')
if sys.version_info.major > 2:
import io
opened = io.TextIOWrapper(opened, newline="")
reader = csv.reader(opened, delimiter=' ')
for splits in reader:
splits[-1] = float(splits[-1])
yield splits
def _split_scores(score_lines, real_id_index, claimed_id_index = 0, score_index = -1):
"""Take the output of :py:func:`four_column` or :py:func:`five_column` and return negatives and positives.
"""
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment