Commit f7e97588 authored by Tim Laibacher's avatar Tim Laibacher

Initial commit

parents
Pipeline #29010 passed with stages
in 13 minutes and 45 seconds
### Bob defaults ###
*~
*.swp
*.pyc
*.so
*.dylib
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
develop-eggs
sphinx
dist
.nfs*
.gdb_history
build
*.egg
src/
record.txt
core
### JupyterNotebook ###
.ipynb_checkpoints
*/.ipynb_checkpoints/*
### VisualStudioCode ###
.vscode/*
.vscode/settings.json
.vscode/tasks.json
.vscode/launch.json
.vscode/extensions.json
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
\ No newline at end of file
include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/single-package.yaml'
\ No newline at end of file
Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/
Written by Tim Laibacher <tim.laibacher@idiap.ch>
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
include README.rst buildout.cfg LICENSE version.txt requirements.txt
recursive-include doc *.rst *.png *.ico *.txt
recursive-include bob *.json *.csv
\ No newline at end of file
.. -*- coding: utf-8 -*-
.. image:: https://img.shields.io/badge/docs-stable-yellow.svg
:target: http://beatubulatest.lab.idiap.ch/private/docs/bob/bob.db.chasedb1/stable/index.html
.. image:: https://img.shields.io/badge/docs-latest-orange.svg
:target: http://beatubulatest.lab.idiap.ch/private/docs/bob/bob.db.chasedb1/master/index.html
.. image:: https://gitlab.idiap.ch/bob/bob.db.chasedb1/badges/master/build.svg
:target: https://gitlab.idiap.ch/bob/bob.db.chasedb1/commits/master
.. image:: https://gitlab.idiap.ch/bob/bob.db.chasedb1/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/bob/bob.db.chasedb1/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/bob/bob.db.chasedb1
.. image:: https://img.shields.io/pypi/v/bob.db.chasedb1.svg
:target: https://pypi.python.org/pypi/bob.db.chasedb1
====================================
CHASE_DB1 Database Interface for Bob
====================================
This package is part of the signal-processing and machine learning toolbox
Bob_. It provides an interface for the `CHASE_DB1 Dataset`_. This package does
not contain the original data files, which need to be obtained through the link
above.
Installation
------------
Complete bob's `installation`_ instructions. Then, to install this
package, run::
$ conda install bob.db.chasedb1
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. Place your references here:
.. _bob: https://www.idiap.ch/software/bob
.. _installation: https://www.idiap.ch/software/bob/install
.. _mailing list: https://www.idiap.ch/software/bob/discuss
.. _chase_db1 dataset: https://blogs.kingston.ac.uk/retinal/chasedb1/
\ No newline at end of file
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""
CHASE_DB1: Retinal image database
Fraz, M.M, Remagnino, P., Hoppe, A., Uyyanonvara, B., Rudnicka, A.R., Owen, C.G. and Barman S.A. (2012) An ensemble classification-based approach applied to retinal blood vessel segmentation. IEEE Transactions on Biomedical Engineering, 59(9), pp. 2538-2548.
https://blogs.kingston.ac.uk/retinal/chasedb1/
"""
from .query import Database
def get_config():
"""Returns a string containing the configuration information.
"""
import bob.extension
return bob.extension.get_config(__name__)
# gets sphinx autodoc done right - don't remove it
__all__ = [_ for _ in dir() if not _.startswith('_')]
{
"train": [
[
"Image_11L.jpg",
"Image_11L_1stHO.png"
],
[
"Image_11R.jpg",
"Image_11R_1stHO.png"
],
[
"Image_12L.jpg",
"Image_12L_1stHO.png"
],
[
"Image_12R.jpg",
"Image_12R_1stHO.png"
],
[
"Image_13L.jpg",
"Image_13L_1stHO.png"
],
[
"Image_13R.jpg",
"Image_13R_1stHO.png"
],
[
"Image_14L.jpg",
"Image_14L_1stHO.png"
],
[
"Image_14R.jpg",
"Image_14R_1stHO.png"
]
],
"test": [
[
"Image_01L.jpg",
"Image_01L_1stHO.png"
],
[
"Image_01R.jpg",
"Image_01R_1stHO.png"
],
[
"Image_02L.jpg",
"Image_02L_1stHO.png"
],
[
"Image_02R.jpg",
"Image_02R_1stHO.png"
],
[
"Image_03L.jpg",
"Image_03L_1stHO.png"
],
[
"Image_03R.jpg",
"Image_03R_1stHO.png"
],
[
"Image_04L.jpg",
"Image_04L_1stHO.png"
],
[
"Image_04R.jpg",
"Image_04R_1stHO.png"
],
[
"Image_05L.jpg",
"Image_05L_1stHO.png"
],
[
"Image_05R.jpg",
"Image_05R_1stHO.png"
],
[
"Image_06L.jpg",
"Image_06L_1stHO.png"
],
[
"Image_06R.jpg",
"Image_06R_1stHO.png"
],
[
"Image_07L.jpg",
"Image_07L_1stHO.png"
],
[
"Image_07R.jpg",
"Image_07R_1stHO.png"
],
[
"Image_08L.jpg",
"Image_08L_1stHO.png"
],
[
"Image_08R.jpg",
"Image_08R_1stHO.png"
],
[
"Image_09L.jpg",
"Image_09L_1stHO.png"
],
[
"Image_09R.jpg",
"Image_09R_1stHO.png"
],
[
"Image_10L.jpg",
"Image_10L_1stHO.png"
],
[
"Image_10R.jpg",
"Image_10R_1stHO.png"
]
]
}
\ No newline at end of file
Image_01L.jpg,Image_01L_1stHO.png
Image_01R.jpg,Image_01R_1stHO.png
Image_02L.jpg,Image_02L_1stHO.png
Image_02R.jpg,Image_02R_1stHO.png
Image_03L.jpg,Image_03L_1stHO.png
Image_03R.jpg,Image_03R_1stHO.png
Image_04L.jpg,Image_04L_1stHO.png
Image_04R.jpg,Image_04R_1stHO.png
Image_05L.jpg,Image_05L_1stHO.png
Image_05R.jpg,Image_05R_1stHO.png
Image_06L.jpg,Image_06L_1stHO.png
Image_06R.jpg,Image_06R_1stHO.png
Image_07L.jpg,Image_07L_1stHO.png
Image_07R.jpg,Image_07R_1stHO.png
Image_08L.jpg,Image_08L_1stHO.png
Image_08R.jpg,Image_08R_1stHO.png
Image_09L.jpg,Image_09L_1stHO.png
Image_09R.jpg,Image_09R_1stHO.png
Image_10L.jpg,Image_10L_1stHO.png
Image_10R.jpg,Image_10R_1stHO.png
\ No newline at end of file
Image_11L.jpg,Image_11L_1stHO.png
Image_11R.jpg,Image_11R_1stHO.png
Image_12L.jpg,Image_12L_1stHO.png
Image_12R.jpg,Image_12R_1stHO.png
Image_13L.jpg,Image_13L_1stHO.png
Image_13R.jpg,Image_13R_1stHO.png
Image_14L.jpg,Image_14L_1stHO.png
Image_14R.jpg,Image_14R_1stHO.png
\ No newline at end of file
from pathlib import Path
import logging
import bob.extension
from .query import Database
def checkfiles(args):
"""
Check if the local database tree is consistent with the expected structure
"""
db = Database(args.protocol)
# go through all files, check if they are available on the filesystem
good = []
bad = []
datadir = Path(args.datadir)
paths = db.paths
for obj in paths:
obj = datadir.joinpath(obj)
if obj.exists():
good.append(obj)
logging.info(f'Found file {obj}')
else:
bad.append(obj)
logging.warning(f'Cannot find file {obj}')
# report
if bad:
logging.warning(f'{len(bad)} files (out of {len(paths)}) were not found at "{str(datadir)}" ')
else:
print('checkfiles completed sucessfully')
# return 0 for self-test
return 0
def add_command(subparsers):
"""Add specific subcommands that the action 'checkfiles' can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('checkfiles', help='Check if the local database tree is consistent with the expected structure')
parser.add_argument('-D', '--datadir', metavar='DIR', default=bob.extension.rc['bob.db.chasedb1.datadir'], help='path to CHASE_DB1 dataset, default uses bob config settings (to check use: bob config show).')
parser.add_argument('-P', '--protocol', metavar='STR', default='default',help='The train/test set protocol to use')
parser.add_argument('--self-test', dest="selftest", default=False, action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) # action
\ No newline at end of file
from pathlib import Path
import csv
import json
import pkg_resources
import bob.extension
def get_file_names(datadir):
"""
Get absolute path of files in a data directory (recursively).
Parameters
----------
data_dir : a Path object
Returns:
-------
file_path: list
a sorted list of paths to all files in the given data directory. Ignores hidden files (suchs as .DS_Store)
"""
file_paths = sorted(list(datadir.glob('**/*?.*')))
rel_file_paths = [f.relative_to(datadir) for f in file_paths]
return rel_file_paths
def read_split(file):
"""
Reads a csv file with two columns: image_file_name, ground_truth_file_name.
Returns
-------
file_list : list
list containing two tuples. The first one contains image file names, the second the ground truth file names
"""
file = open(file,"r")
rows = csv.reader(file)
file_list = list(zip(*rows))
file.close()
return file_list
def create(args):
"""
Writes a 'database' json file to disk
Parameters
----------
args.train : csv containg train img, gt pairs
args.test : csv containg test img,gt pairs
args.datadir : path to the dataset on disk
args.output : name of the output file
Returns
-------
int : 0
"""
# read file name pairs as defined in the csv
basedir = Path(pkg_resources.resource_filename(__name__, ''))
train_split = read_split(basedir.joinpath(args.train))
test_split = read_split(basedir.joinpath(args.test))
# get actual file paths and create dictionary (filname, path)
files = get_file_names(Path(args.datadir))
filedict = dict([(f.name, f) for f in files])
# map split to files
train_images = [str(filedict[img]) for img in train_split[0]]
train_ground_truths = [str(filedict[gt]) for gt in train_split[1]]
test_images = [str(filedict[img]) for img in test_split[0]]
test_ground_truths = [str(filedict[gt]) for gt in test_split[1]]
# sense check
assert len(train_images) == len(train_ground_truths)
assert len(test_images) == len(test_ground_truths)
# zip together and create dict
train = list(zip(train_images,train_ground_truths))
test = list(zip(test_images,test_ground_truths))
db = {'train':train,'test':test}
# json
with open(basedir.joinpath(args.output),'w') as outfile:
json.dump(db, outfile, indent = 1)
# return 0 for self-test
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "create" can use"""
from argparse import SUPPRESS
parser = subparsers.add_parser('create', help='Creates json file with train and test splits')
parser.add_argument('-D', '--datadir', metavar='DIR', default=bob.extension.rc['bob.db.chasedb1.datadir'], help='path to CHASE_DB1 dataset, default uses bob config settings (to check use: bob config show).')
parser.add_argument('-T', '--train', metavar='STR', default='chasedb1_train.csv', help='csv containg train img,gt pairs')
parser.add_argument('-E', '--test', metavar='STR', default='chasedb1_test.csv', help='csv containg test img,gt pairs')
parser.add_argument('-O', '--output', metavar='STR', default='chasedb1_db_default.json', help='output file name')
parser.add_argument('--self-test', dest="selftest", default=False, action='store_true', help=SUPPRESS)
parser.set_defaults(func=create) #action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
"""Commands the database can respond to.
"""
import pkg_resources
from bob.db.base.driver import Interface as BaseInterface
class Interface(BaseInterface):
def name(self):
return 'chasedb1'
def version(self):
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def files(self):
"""
List of meta-data files for the package to be downloaded/uploaded
This function should normally return an empty list, except in case the
database being implemented requires download/upload of metadata files that
are **not** kept in its (git) repository.
"""
return []
def type(self):
return 'text'
def add_commands(self, parser):
from . import __doc__ as docs
subparsers = self.setup_parser(parser,"CHASE_DB1 database", docs)
# checkfiles command
from .checkfiles import add_command as checkfiles_command
checkfiles_command(subparsers)
# create command
from .create import add_command as create_command
create_command(subparsers)
\ No newline at end of file
from pathlib import Path
from PIL import Image
import bob.extension
class FundusImage:
"""
Generic fundus image object.
"""
def __init__(self, path):
self.path = path
@property
def basename(self):
"""
Returns the file name
Returns
-------
name : str
"""
return Path(self.path).name
@property
def size(self):
"""
Returns
-------
size : tuple
the fundus image resolution in (W, H).
"""
return self.pil_image().size
def pil_image(self, datadir=bob.extension.rc['bob.db.chasedb1.datadir']):
"""
Returns
-------
img : :py:class:`PIL.Image.Image`
"""
img = Image.open(str(Path(datadir).joinpath(self.path)))
return img
class GroundTruth:
"""
Generic ground truth object.
- Allows for thresholding in case there are multiple ground truth annotations in one bitmap
- Allows for "on-the-fly" drawing of ground truth annotations with a specified size
Parameters
----------
path : str
relative path to the file on disk
threshold : float
in range [0,1] used to threshold ground-truth image data with multiple classes e.g. optic disc and cup
drawsize : tuple
size tuple in pillow (W, H) format. Used for cases where drawing of gt is required
"""
def __init__(self, path, threshold=None, drawsize=None):
self.path = path
self.threshold = threshold
self.drawsize = drawsize
@property
def basename(self):
"""
Returns the file name
Returns
-------
name : str
"""
return Path(self.path).name
@property
def size(self):
"""
Retirms the ground truth image resolution in (W, H).
Returns
-------
size : tuple
"""
return self.pil_image().size
def pil_image(self, datadir=bob.extension.rc['bob.db.chasedb1.datadir']):
"""
Returns
-------
gt : :py:class:`PIL.Image.Image`
mode = '1'
"""
gt = Image.open(str(Path(datadir).joinpath(self.path)))
gt = gt.convert(mode='1', dither=None)
return gt
class Sample:
"""
Generic Sample object
High level sample class that combines the objects 'FundusImage' and 'GroundTruth'
Allows for access of the subclass, e.g. :
.. testsetup:: *
from bob.db.chasedb1.models import *
.. doctest::
>>> img = FundusImage('path/to/some_img.file')
>>> gt = GroundTruth('path/to/some_gt.file')
>>> mysample = Sample(img, gt)
>>> mysample.img.basename
'some_img.file'
>>> mysample.gt.basename
'some_gt.file'
>>>
Parameters
----------
img : FundusImage
gt : GroundTruth
"""
def __init__(self, img, gt):
self.img = img
self.gt = gt
@property
def paths(self):
"""
Returns
--------
paths : list
paths of image, ground truth
"""
return self.img.path, self.gt.path
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
from PIL import Image
import json
from pathlib import Path
import pkg_resources
import itertools
from .models import Sample, FundusImage, GroundTruth
class FileList:
"""
FileList object that loads the protocol as defined in a json file.
Provides a ``__getitem__`` interface.
"""
def __init__(self, db_json):
with open(db_json,'r') as in_file:
self._filelist = json.load(in_file)
def __getitem__(self, key):
# if no valid split is passed, return all paths of train and test
return self._filelist.get(key, (self._filelist['train']+self._filelist['test']))
class Database:
"""
A low level database interface to be used with PyTorch or other deep learning frameworks.
Parameters
----------
protocol : str
protocol defining the train-test split.
"""
def __init__(self, protocol = 'default'):
self.protocol = protocol
root = Path(pkg_resources.resource_filename(__name__, ''))
db_json = root.joinpath('chasedb1_db_'+self.protocol+'.json')
# initialize filelist
self._filelist = FileList(db_json)
@property
def paths(self):
"""
Returns
-------
paths : list
list of all paths of all samples
"""
return list(itertools.chain(*(self._filelist[None]) ))
def _make_sample(self, img_path,gt_path):
"""
Make a single sample object
Parameters
----------
img_path : str
relative path to image
gt_path : str
relative path to ground truth
Returns
-------
sample : Sample
"""
img = FundusImage(img_path)
gt = GroundTruth(gt_path)
return Sample(img, gt)
def samples(self, split=None):
"""
Given a split, returns a list of Sample objects.
Parameters
----------
split : str
'train', 'test' or None (returns all samples)
Returns
-------
samples : list
list of Sample objects
"""
samples = []
for s in self._filelist[split]:
sample_obj = self._make_sample(s[0], s[1])
samples.append(sample_obj)