FingerCrop.py 17.1 KB
Newer Older
Pedro TOME's avatar
Pedro TOME committed
1 2
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
3 4 5 6

import math
import numpy
from PIL import Image
Pedro TOME's avatar
Pedro TOME committed
7 8 9

import bob.io.base

10 11
from bob.bio.base.preprocessor import Preprocessor

Pedro TOME's avatar
Pedro TOME committed
12
from .. import utils
13

Pedro TOME's avatar
Pedro TOME committed
14 15

class FingerCrop (Preprocessor):
Olegs NIKISINS's avatar
Olegs NIKISINS committed
16
  """
17
  Extracts the mask heuristically and pre-processes fingervein images.
18 19 20 21 22

  Based on the implementation: E.C. Lee, H.C. Lee and K.R. Park. Finger vein
  recognition using minutia-based alignment and local binary pattern-based
  feature extraction. International Journal of Imaging Systems and
  Technology. Vol. 19, No. 3, pp. 175-178, September 2009.
23

24 25 26 27 28 29 30 31 32
  Finger orientation is based on B. Huang, Y. Dai, R. Li, D. Tang and W. Li,
  Finger-vein authentication based on wide line detector and pattern
  normalization, Proceedings on 20th International Conference on Pattern
  Recognition (ICPR), 2010.

  The ``konomask`` option is based on the work of M. Kono, H. Ueki and S.
  Umemura. Near-infrared finger vein patterns for personal identification,
  Applied Optics, Vol. 41, Issue 35, pp. 7429-7436 (2002).

33 34
  In this implementation, the finger image is (in this order):

35 36 37 38 39 40 41 42 43 44 45
    1. The mask is extracted (if ``annotation`` is not chosen as a parameter to
       ``fingercontour``). Other mask extraction options correspond to
       heuristics developed by Lee et al. (2009) or Kono et al. (2002)
    2. The finger is normalized (made horizontal), via a least-squares
       normalization procedure concerning the center of the annotated area,
       width-wise. Before normalization, the image is padded to avoid loosing
       pixels corresponding to veins during the rotation
    3. (optionally) Post processed with histogram-equalization to enhance vein
       information. Notice that only the area inside the mask is used for
       normalization. Areas outside of the mask (where the mask is ``False``
       are set to black)
46

47

48
  Parameters:
49

50 51 52
    mask_h (:py:obj:`int`, optional): Height of contour mask in pixels, must
      be an even number (used by the methods ``leemaskMod`` or
      ``leemaskMatlab``)
53

54 55
    mask_w (:py:obj:`int`, optional): Width of the contour mask in pixels
      (used by the methods ``leemaskMod`` or ``leemaskMatlab``)
56

57 58 59 60
    padding_width (:py:obj:`int`, optional): How much padding (in pixels) to
      add around the borders of the input image. We normally always keep this
      value on its default (5 pixels). This parameter is always used before
      normalizing the finger orientation.
61

62 63 64
    padding_constant (:py:obj:`int`, optional): What is the value of the pixels
      added to the padding. This number should be a value between 0 and 255.
      (From Pedro Tome: for UTFVP (high-quality samples), use 0. For the VERA
65
      Fingervein database (low-quality samples), use 51 (that corresponds to
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
      0.2 in a float image with values between 0 and 1). This parameter is
      always used before normalizing the finger orientation.

    fingercontour (:py:obj:`str`, optional): Select between three finger
      contour implementations: ``"leemaskMod"``, ``"leemaskMatlab"``,
      ``"konomask"`` or ``annotation``. (From Pedro Tome: the option
      ``leemaskMatlab`` was just implemented for testing purposes so we could
      compare with MAT files generated from Matlab code of other authors. He
      only used it with the UTFVP database, using ``leemaskMod`` with that
      database yields slight worse results.)

    postprocessing (:py:obj:`str`, optional): Select between ``HE`` (histogram
      equalization, as with :py:func:`skimage.exposure.equalize_hist`) or
      ``None`` (the default).

81
  """
Pedro TOME's avatar
Pedro TOME committed
82

83

84 85 86
  def __init__(self, mask_h = 4, mask_w = 40,
      padding_width = 5, padding_constant = 51,
      fingercontour = 'leemaskMod', postprocessing = None, **kwargs):
Pedro TOME's avatar
Pedro TOME committed
87

88
    Preprocessor.__init__(self,
Pedro TOME's avatar
Pedro TOME committed
89 90
        mask_h = mask_h,
        mask_w = mask_w,
91 92
        padding_width = padding_width,
        padding_constant = padding_constant,
Pedro TOME's avatar
Pedro TOME committed
93 94 95
        fingercontour = fingercontour,
        postprocessing = postprocessing,
        **kwargs
96
        )
Pedro TOME's avatar
Pedro TOME committed
97 98 99

    self.mask_h = mask_h
    self.mask_w = mask_w
100

Pedro TOME's avatar
Pedro TOME committed
101 102 103
    self.fingercontour = fingercontour
    self.postprocessing = postprocessing

104 105
    self.padding_width = padding_width
    self.padding_constant = padding_constant
106 107


Pedro TOME's avatar
Pedro TOME committed
108
  def __konomask__(self, image, sigma):
Olegs NIKISINS's avatar
Olegs NIKISINS committed
109 110
    """
    Finger vein mask extractor.
111 112 113 114 115

    Based on the work of M. Kono, H. Ueki and S. Umemura. Near-infrared finger
    vein patterns for personal identification, Applied Optics, Vol. 41, Issue
    35, pp. 7429-7436 (2002).

116 117
    """

Pedro TOME's avatar
Pedro TOME committed
118 119
    sigma = 5
    img_h,img_w = image.shape
120

Pedro TOME's avatar
Pedro TOME committed
121 122 123 124 125 126 127 128 129 130 131 132
    # Determine lower half starting point
    if numpy.mod(img_h,2) == 0:
        half_img_h = img_h/2 + 1
    else:
        half_img_h = numpy.ceil(img_h/2)

    #Construct filter kernel
    winsize = numpy.ceil(4*sigma)

    x = numpy.arange(-winsize, winsize+1)
    y = numpy.arange(-winsize, winsize+1)
    X, Y = numpy.meshgrid(x, y)
133

Pedro TOME's avatar
Pedro TOME committed
134
    hy = (-Y/(2*math.pi*sigma**4))*numpy.exp(-(X**2 + Y**2)/(2*sigma**2))
135

Pedro TOME's avatar
Pedro TOME committed
136
    # Filter the image with the directional kernel
137
    fy = utils.imfilter(image, hy)
Pedro TOME's avatar
Pedro TOME committed
138 139 140 141 142 143 144 145

    # Upper part of filtred image
    img_filt_up = fy[0:half_img_h,:]
    y_up = img_filt_up.argmax(axis=0)

    # Lower part of filtred image
    img_filt_lo = fy[half_img_h-1:,:]
    y_lo = img_filt_lo.argmin(axis=0)
146 147 148

    # Fill region between upper and lower edges
    finger_mask = numpy.ndarray(image.shape, numpy.bool)
Pedro TOME's avatar
Pedro TOME committed
149 150 151 152
    finger_mask[:,:] = False

    for i in range(0,img_w):
      finger_mask[y_up[i]:y_lo[i]+image.shape[0]-half_img_h+2,i] = True
153

154
    return finger_mask
Pedro TOME's avatar
Pedro TOME committed
155

156

Pedro TOME's avatar
Pedro TOME committed
157
  def __leemaskMod__(self, image):
Olegs NIKISINS's avatar
Olegs NIKISINS committed
158 159
    """
    A method to calculate the finger mask.
160

161 162 163 164
    Based on the work of Finger vein recognition using minutia-based alignment
    and local binary pattern-based feature extraction, E.C. Lee, H.C. Lee and
    K.R. Park, International Journal of Imaging Systems and Technology, Volume
    19, Issue 3, September 2009, Pages 175--178, doi: 10.1002/ima.20193
165

166 167
    This code is a variant of the Matlab implementation by Bram Ton, available
    at:
168

169
    https://nl.mathworks.com/matlabcentral/fileexchange/35752-finger-region-localisation/content/lee_region.m
Pedro TOME's avatar
Pedro TOME committed
170

171 172 173 174
    In this variant from Pedro Tome, the technique of filtering the image with
    a horizontal filter is also applied on the vertical axis.


175
    Parameters:
176

Olegs NIKISINS's avatar
Olegs NIKISINS committed
177
    image (numpy.ndarray): raw image to use for finding the mask, as 2D array
178 179 180
        of unsigned 8-bit integers


Olegs NIKISINS's avatar
Olegs NIKISINS committed
181
    **Returns:**
182

Olegs NIKISINS's avatar
Olegs NIKISINS committed
183
    numpy.ndarray: A 2D boolean array with the same shape of the input image
184 185
        representing the cropping mask. ``True`` values indicate where the
        finger is.
Pedro TOME's avatar
Pedro TOME committed
186

Olegs NIKISINS's avatar
Olegs NIKISINS committed
187
    numpy.ndarray: A 2D array with 64-bit floats indicating the indexes where
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
       the mask, for each column, starts and ends on the original image. The
       same of this array is (2, number of columns on input image).

    """


    img_h,img_w = image.shape

    # Determine lower half starting point
    half_img_h = img_h/2
    half_img_w = img_w/2

    # Construct mask for filtering (up-bottom direction)
    mask = numpy.ones((self.mask_h, self.mask_w), dtype='float64')
    mask[(self.mask_h/2):,:] = -1.0

    img_filt = utils.imfilter(image, mask)
205

Pedro TOME's avatar
Pedro TOME committed
206
    # Upper part of filtred image
207
    img_filt_up = img_filt[:half_img_h,:]
Pedro TOME's avatar
Pedro TOME committed
208
    y_up = img_filt_up.argmax(axis=0)
209

210 211
    # Lower part of filtred image
    img_filt_lo = img_filt[half_img_h:,:]
Pedro TOME's avatar
Pedro TOME committed
212
    y_lo = img_filt_lo.argmin(axis=0)
213

214
    img_filt = utils.imfilter(image, mask.T)
215

216 217
    # Left part of filtered image
    img_filt_lf = img_filt[:,:half_img_w]
Pedro TOME's avatar
Pedro TOME committed
218
    y_lf = img_filt_lf.argmax(axis=1)
219

220
    # Right part of filtred image
Pedro TOME's avatar
Pedro TOME committed
221 222
    img_filt_rg = img_filt[:,half_img_w:]
    y_rg = img_filt_rg.argmin(axis=1)
223

224
    finger_mask = numpy.zeros(image.shape, dtype='bool')
225

Pedro TOME's avatar
Pedro TOME committed
226 227
    for i in range(0,y_up.size):
        finger_mask[y_up[i]:y_lo[i]+img_filt_lo.shape[0]+1,i] = True
228

Pedro TOME's avatar
Pedro TOME committed
229 230 231
    # Left region
    for i in range(0,y_lf.size):
        finger_mask[i,0:y_lf[i]+1] = False
232

233 234
    # Right region has always the finger ending, crop the padding with the
    # meadian
235
    finger_mask[:,int(numpy.median(y_rg)+img_filt_rg.shape[1]):] = False
236

237
    return finger_mask
238 239


Pedro TOME's avatar
Pedro TOME committed
240
  def __leemaskMatlab__(self, image):
Olegs NIKISINS's avatar
Olegs NIKISINS committed
241 242
    """
    A method to calculate the finger mask.
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262

    Based on the work of Finger vein recognition using minutia-based alignment
    and local binary pattern-based feature extraction, E.C. Lee, H.C. Lee and
    K.R. Park, International Journal of Imaging Systems and Technology, Volume
    19, Issue 3, September 2009, Pages 175--178, doi: 10.1002/ima.20193

    This code is based on the Matlab implementation by Bram Ton, available at:

    https://nl.mathworks.com/matlabcentral/fileexchange/35752-finger-region-localisation/content/lee_region.m

    In this method, we calculate the mask of the finger independently for each
    column of the input image. Firstly, the image is convolved with a [1,-1]
    filter of size ``(self.mask_h, self.mask_w)``. Then, the upper and lower
    parts of the resulting filtered image are separated. The location of the
    maxima in the upper part is located. The same goes for the location of the
    minima in the lower part. The mask is then calculated, per column, by
    considering it starts in the point where the maxima is in the upper part
    and goes up to the point where the minima is detected on the lower part.


Olegs NIKISINS's avatar
Olegs NIKISINS committed
263
    **Parameters:**
264

Olegs NIKISINS's avatar
Olegs NIKISINS committed
265
    image (numpy.ndarray): raw image to use for finding the mask, as 2D array
266 267 268
        of unsigned 8-bit integers


Olegs NIKISINS's avatar
Olegs NIKISINS committed
269
    **Returns:**
270

Olegs NIKISINS's avatar
Olegs NIKISINS committed
271
    numpy.ndarray: A 2D boolean array with the same shape of the input image
272 273 274
        representing the cropping mask. ``True`` values indicate where the
        finger is.

Olegs NIKISINS's avatar
Olegs NIKISINS committed
275
    numpy.ndarray: A 2D array with 64-bit floats indicating the indexes where
276 277 278 279
       the mask, for each column, starts and ends on the original image. The
       same of this array is (2, number of columns on input image).

    """
Pedro TOME's avatar
Pedro TOME committed
280 281

    img_h,img_w = image.shape
282

Pedro TOME's avatar
Pedro TOME committed
283
    # Determine lower half starting point
284
    half_img_h = int(img_h/2)
285 286

    # Construct mask for filtering
287
    mask = numpy.ones((self.mask_h,self.mask_w), dtype='float64')
288
    mask[int(self.mask_h/2):,:] = -1.0
Pedro TOME's avatar
Pedro TOME committed
289

290
    img_filt = utils.imfilter(image, mask)
291

292 293
    # Upper part of filtered image
    img_filt_up = img_filt[:half_img_h,:]
Pedro TOME's avatar
Pedro TOME committed
294 295
    y_up = img_filt_up.argmax(axis=0)

296 297
    # Lower part of filtered image
    img_filt_lo = img_filt[half_img_h:,:]
Pedro TOME's avatar
Pedro TOME committed
298
    y_lo = img_filt_lo.argmin(axis=0)
299

300 301 302 303 304 305
    # Translation: for all columns of the input image, set to True all pixels
    # of the mask from index where the maxima occurred in the upper part until
    # the index where the minima occurred in the lower part.
    finger_mask = numpy.zeros(image.shape, dtype='bool')
    for i in range(img_filt.shape[1]):
      finger_mask[y_up[i]:(y_lo[i]+img_filt_lo.shape[0]+1), i] = True
306

307
    return finger_mask
308

Pedro TOME's avatar
Pedro TOME committed
309

310
  def __huangnormalization__(self, image, mask):
Olegs NIKISINS's avatar
Olegs NIKISINS committed
311 312
    """
    Simple finger normalization.
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328

    Based on B. Huang, Y. Dai, R. Li, D. Tang and W. Li, Finger-vein
    authentication based on wide line detector and pattern normalization,
    Proceedings on 20th International Conference on Pattern Recognition (ICPR),
    2010.

    This implementation aligns the finger to the centre of the image using an
    affine transformation. Elliptic projection which is described in the
    referenced paper is not included.

    In order to defined the affine transformation to be performed, the
    algorithm first calculates the center for each edge (column wise) and
    calculates the best linear fit parameters for a straight line passing
    through those points.


Olegs NIKISINS's avatar
Olegs NIKISINS committed
329
    **Parameters:**
330

Olegs NIKISINS's avatar
Olegs NIKISINS committed
331
    image (numpy.ndarray): raw image to normalize as 2D array of unsigned
332 333
        8-bit integers

Olegs NIKISINS's avatar
Olegs NIKISINS committed
334
    mask (numpy.ndarray): mask to normalize as 2D array of booleans
335 336


Olegs NIKISINS's avatar
Olegs NIKISINS committed
337
    **Returns:**
338

Olegs NIKISINS's avatar
Olegs NIKISINS committed
339
    numpy.ndarray: A 2D boolean array with the same shape and data type of
340 341
        the input image representing the newly aligned image.

Olegs NIKISINS's avatar
Olegs NIKISINS committed
342
    numpy.ndarray: A 2D boolean array with the same shape and data type of
343
        the input mask representing the newly aligned mask.
344

345
    """
Pedro TOME's avatar
Pedro TOME committed
346 347

    img_h, img_w = image.shape
348 349 350 351 352 353 354

    if self.padding_width:
      mask_consider = mask[self.padding_width:-self.padding_width,
          self.padding_width:-self.padding_width]
    else:
      mask_consider = mask

355
    n_edges = mask_consider.shape[1]
Pedro TOME's avatar
Pedro TOME committed
356

357
    # Calculates the mask edges along the columns
358 359 360 361
    edges = numpy.zeros((2, n_edges), dtype=int)

    edges[0,:] = mask_consider.argmax(axis=0) # get upper edges
    edges[1,:] = len(mask) - numpy.flipud(mask_consider).argmax(axis=0) - 1
362

363
    bl = edges.mean(axis=0) #baseline
Pedro TOME's avatar
Pedro TOME committed
364 365
    x = numpy.arange(0,img_w)
    A = numpy.vstack([x, numpy.ones(len(x))]).T
366

Pedro TOME's avatar
Pedro TOME committed
367 368
    # Fit a straight line through the base line points
    w = numpy.linalg.lstsq(A,bl)[0] # obtaining the parameters
369

Pedro TOME's avatar
Pedro TOME committed
370 371 372
    angle = -1*math.atan(w[0])  # Rotation
    tr = img_h/2 - w[1]         # Translation
    scale = 1.0                 # Scale
373 374

    #Affine transformation parameters
Pedro TOME's avatar
Pedro TOME committed
375 376 377
    sx=sy=scale
    cosine = math.cos(angle)
    sine = math.sin(angle)
378

Pedro TOME's avatar
Pedro TOME committed
379 380 381 382
    a = cosine/sx
    b = -sine/sy
    #b = sine/sx
    c = 0 #Translation in x
383

Pedro TOME's avatar
Pedro TOME committed
384 385 386 387 388
    d = sine/sx
    e = cosine/sy
    f = tr #Translation in y
    #d = -sine/sy
    #e = cosine/sy
389 390 391 392 393 394 395
    #f = 0

    g = 0
    h = 0
    #h=tr
    i = 1

Pedro TOME's avatar
Pedro TOME committed
396 397 398
    T = numpy.matrix([[a,b,c],[d,e,f],[g,h,i]])
    Tinv = numpy.linalg.inv(T)
    Tinvtuple = (Tinv[0,0],Tinv[0,1], Tinv[0,2], Tinv[1,0],Tinv[1,1],Tinv[1,2])
399

Pedro TOME's avatar
Pedro TOME committed
400
    img=Image.fromarray(image)
401 402
    image_norm = img.transform(img.size, Image.AFFINE, Tinvtuple,
        resample=Image.BICUBIC)
403
    image_norm = numpy.array(image_norm)
Pedro TOME's avatar
Pedro TOME committed
404 405

    finger_mask = numpy.zeros(mask.shape)
406
    finger_mask[mask] = 1
Pedro TOME's avatar
Pedro TOME committed
407 408

    img_mask=Image.fromarray(finger_mask)
409 410
    mask_norm = img_mask.transform(img_mask.size, Image.AFFINE, Tinvtuple,
        resample=Image.BICUBIC)
411
    mask_norm = numpy.array(mask_norm).astype('bool')
412

413
    return (image_norm, mask_norm)
414

Pedro TOME's avatar
Pedro TOME committed
415

416
  def __HE__(self, image, mask):
Olegs NIKISINS's avatar
Olegs NIKISINS committed
417 418
    """
    Applies histogram equalization on the input image inside the mask.
419 420 421 422

    In this implementation, only the pixels that lie inside the mask will be
    used to calculate the histogram equalization parameters. Because of this
    particularity, we don't use Bob's implementation for histogram equalization
423
    and have one based exclusively on scikit-image.
Pedro TOME's avatar
Pedro TOME committed
424

425

Olegs NIKISINS's avatar
Olegs NIKISINS committed
426
    **Parameters:**
427

Olegs NIKISINS's avatar
Olegs NIKISINS committed
428
    image (numpy.ndarray): raw image to be filtered, as 2D array of
429 430
          unsigned 8-bit integers

Olegs NIKISINS's avatar
Olegs NIKISINS committed
431
    mask (numpy.ndarray): mask of the same size of the image, but composed
432 433 434
          of boolean values indicating which values should be considered for
          the histogram equalization

435

Olegs NIKISINS's avatar
Olegs NIKISINS committed
436
    **Returns:**
437

Olegs NIKISINS's avatar
Olegs NIKISINS committed
438
    numpy.ndarray: normalized image as a 2D array of unsigned 8-bit integers
439

440
    """
441
    from skimage.exposure import equalize_hist
442

443
    retval = equalize_hist(image, mask=mask)
444

445 446
    # make the parts outside the mask totally black
    retval[~mask] = 0
447

448
    return retval
Pedro TOME's avatar
Pedro TOME committed
449

450

451
  def __call__(self, data, annotations=None):
452
    """Reads the input image or (image, mask) and prepares for fex.
453

454
    Parameters:
Pedro TOME's avatar
Pedro TOME committed
455

456 457 458 459 460
      data (numpy.ndarray, tuple): Either a :py:class:`numpy.ndarray`
        containing a gray-scaled image with dtype ``uint8`` or a 2-tuple
        containing both the gray-scaled image and a mask, with the same size of
        the image, with dtype ``bool`` containing the points which should be
        considered part of the finger
Pedro TOME's avatar
Pedro TOME committed
461

462

463
    Returns:
Pedro TOME's avatar
Pedro TOME committed
464

465
      numpy.ndarray: The image, preprocessed and normalized
Pedro TOME's avatar
Pedro TOME committed
466

467 468
      numpy.ndarray: A mask, of the same size of the image, indicating where
      the valid data for the object is.
469

Olegs NIKISINS's avatar
Olegs NIKISINS committed
470
    """
471

472 473 474 475 476
    if isinstance(data, numpy.ndarray):
      image = data
      mask = None
    else:
      image, mask = data
477

478 479 480
    # 1. Pads the input image if any padding should be added
    image = numpy.pad(image, self.padding_width, 'constant',
        constant_values = self.padding_constant)
481 482

    ## Finger edges and contour extraction:
Pedro TOME's avatar
Pedro TOME committed
483
    if self.fingercontour == 'leemaskMatlab':
484
      mask = self.__leemaskMatlab__(image) #for UTFVP
485
    elif self.fingercontour == 'leemaskMod':
486
      mask = self.__leemaskMod__(image) #for VERA
487
    elif self.fingercontour == 'konomask':
488 489 490 491 492
      mask = self.__konomask__(image, sigma=5)
    elif self.fingercontour == 'annotation':
      if mask is None:
        raise RuntimeError("Cannot use fingercontour=annotation - the " \
            "current sample being processed does not provide a mask")
493 494 495
      # Pads the mask to ensure both things are the same in size
      mask = numpy.pad(mask, self.padding_width, 'constant',
          constant_values = self.padding_constant)
496 497 498 499
    else:
      raise RuntimeError("Please choose between leemaskMod, leemaskMatlab, " \
          "konomask or annotation for parameter 'fingercontour'. %s is not " \
          "valid" % self.fingercontour)
500

Pedro TOME's avatar
Pedro TOME committed
501
    ## Finger region normalization:
502
    image_norm, mask_norm = self.__huangnormalization__(image, mask)
Pedro TOME's avatar
Pedro TOME committed
503

504
    ## veins enhancement:
505
    if self.postprocessing == 'HE':
506
      image_norm = self.__HE__(image_norm, mask_norm)
507

508
    ## returns the normalized image and the finger mask
509
    return image_norm, mask_norm
Pedro TOME's avatar
Pedro TOME committed
510 511


512 513
  def write_data(self, data, filename):
    '''Overrides the default method implementation to handle our tuple'''
Pedro TOME's avatar
Pedro TOME committed
514

515 516
    f = bob.io.base.HDF5File(filename, 'w')
    f.set('image', data[0])
517
    f.set('mask', data[1])
518

Pedro TOME's avatar
Pedro TOME committed
519

520 521
  def read_data(self, filename):
    '''Overrides the default method implementation to handle our tuple'''
522

523
    f = bob.io.base.HDF5File(filename, 'r')
524
    return f.read('image'), f.read('mask')