Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
bob.learn.tensorflow
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
11
Issues
11
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
bob
bob.learn.tensorflow
Commits
109eb4c3
Commit
109eb4c3
authored
Oct 16, 2016
by
Tiago de Freitas Pereira
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Implemented data augmentation
parent
bd6430f5
Changes
13
Hide whitespace changes
Inline
Side-by-side
Showing
13 changed files
with
400 additions
and
34 deletions
+400
-34
bob/learn/tensorflow/datashuffler/Base.py
bob/learn/tensorflow/datashuffler/Base.py
+28
-5
bob/learn/tensorflow/datashuffler/DataAugmentation.py
bob/learn/tensorflow/datashuffler/DataAugmentation.py
+33
-0
bob/learn/tensorflow/datashuffler/Disk.py
bob/learn/tensorflow/datashuffler/Disk.py
+12
-3
bob/learn/tensorflow/datashuffler/ImageAugmentation.py
bob/learn/tensorflow/datashuffler/ImageAugmentation.py
+72
-0
bob/learn/tensorflow/datashuffler/Memory.py
bob/learn/tensorflow/datashuffler/Memory.py
+14
-5
bob/learn/tensorflow/datashuffler/SiameseDisk.py
bob/learn/tensorflow/datashuffler/SiameseDisk.py
+5
-2
bob/learn/tensorflow/datashuffler/SiameseMemory.py
bob/learn/tensorflow/datashuffler/SiameseMemory.py
+23
-9
bob/learn/tensorflow/datashuffler/TripletDisk.py
bob/learn/tensorflow/datashuffler/TripletDisk.py
+4
-2
bob/learn/tensorflow/datashuffler/TripletMemory.py
bob/learn/tensorflow/datashuffler/TripletMemory.py
+23
-5
bob/learn/tensorflow/datashuffler/__init__.py
bob/learn/tensorflow/datashuffler/__init__.py
+5
-0
bob/learn/tensorflow/test/test_cnn.py
bob/learn/tensorflow/test/test_cnn.py
+8
-3
bob/learn/tensorflow/test/test_datashuffler_augmentation.py
bob/learn/tensorflow/test/test_datashuffler_augmentation.py
+172
-0
bob/learn/tensorflow/test/test_dnn.py
bob/learn/tensorflow/test/test_dnn.py
+1
-0
No files found.
bob/learn/tensorflow/datashuffler/Base.py
View file @
109eb4c3
...
...
@@ -15,7 +15,8 @@ class Base(object):
input_dtype
=
"float64"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
The class provide base functionalities to shuffle the data before to train a neural network
...
...
@@ -56,6 +57,8 @@ class Base(object):
self
.
data_placeholder
=
None
self
.
label_placeholder
=
None
self
.
data_augmentation
=
data_augmentation
def
get_placeholders
(
self
,
name
=
""
):
"""
Returns a place holder with the size of your batch
...
...
@@ -83,14 +86,34 @@ class Base(object):
Convert bob color image to the skcit image
"""
skimage
=
numpy
.
zeros
(
shape
=
(
bob_image
.
shape
[
1
],
bob_image
.
shape
[
2
],
3
))
skimage
=
numpy
.
zeros
(
shape
=
(
bob_image
.
shape
[
1
],
bob_image
.
shape
[
2
],
bob_image
.
shape
[
0
]
))
skimage
[:,
:,
0
]
=
bob_image
[
0
,
:,
:]
# Copying red
skimage
[:,
:,
1
]
=
bob_image
[
1
,
:,
:]
# Copying green
skimage
[:,
:,
2
]
=
bob_image
[
2
,
:,
:]
# Copying blue
for
i
in
range
(
bob_image
.
shape
[
0
]):
skimage
[:,
:,
i
]
=
bob_image
[
i
,
:,
:]
# Copying red
#skimage[:, :, 1] = bob_image[1, :, :] # Copying green
#skimage[:, :, 2] = bob_image[2, :, :] # Copying blue
return
skimage
def
skimage2bob
(
self
,
sk_image
):
"""
Convert bob color image to the skcit image
"""
bob_image
=
numpy
.
zeros
(
shape
=
(
sk_image
.
shape
[
2
],
sk_image
.
shape
[
0
],
sk_image
.
shape
[
1
]))
for
i
in
range
(
bob_image
.
shape
[
0
]):
bob_image
[
i
,
:,
:]
=
sk_image
[:,
:,
i
]
# Copying red
#bob_image[0, :, :] = sk_image[:, :, 0] # Copying red
#if bob_image.shape[0] > 1:
# bob_image[1, :, :] = sk_image[:, :, 1]# Copying green
# bob_image[2, :, :] = sk_image[:, :, 2] # Copying blue
return
bob_image
def
rescale
(
self
,
data
):
"""
Reescale a single sample with input_shape
...
...
bob/learn/tensorflow/datashuffler/DataAugmentation.py
0 → 100644
View file @
109eb4c3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Sun 16 Oct 2016 14:32:36 CEST
import
numpy
class
DataAugmentation
(
object
):
"""
Base class for applying common real-time data augmentation.
This class is meant to be used as an argument of `input_data`. When training
a model, the defined augmentation methods will be applied at training
time only.
"""
def
__init__
(
self
,
seed
=
10
):
self
.
filter_bank
=
[]
numpy
.
random
.
seed
(
seed
)
def
__call__
(
self
,
image
):
"""
Apply a random filter to and image
"""
if
len
(
self
.
filter_bank
)
<=
0
:
raise
ValueError
(
"There is not filters in the filter bank"
)
filter
=
self
.
filter_bank
[
numpy
.
random
.
randint
(
len
(
self
.
filter_bank
))]
return
filter
(
image
)
bob/learn/tensorflow/datashuffler/Disk.py
View file @
109eb4c3
...
...
@@ -19,7 +19,8 @@ class Disk(Base):
input_dtype
=
"float64"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
This datashuffler deal with databases that are stored in the disk.
The data is loaded on the fly,.
...
...
@@ -46,7 +47,8 @@ class Disk(Base):
input_dtype
=
input_dtype
,
scale
=
scale
,
batch_size
=
batch_size
,
seed
=
seed
seed
=
seed
,
data_augmentation
=
data_augmentation
)
# Seting the seed
numpy
.
random
.
seed
(
seed
)
...
...
@@ -56,6 +58,11 @@ class Disk(Base):
def
load_from_file
(
self
,
file_name
):
d
=
bob
.
io
.
base
.
load
(
file_name
)
# Applying the data augmentation
if
self
.
data_augmentation
is
not
None
:
d
=
self
.
data_augmentation
(
d
)
if
d
.
shape
[
0
]
!=
3
and
self
.
input_shape
[
2
]
!=
3
:
# GRAY SCALE IMAGE
data
=
numpy
.
zeros
(
shape
=
(
d
.
shape
[
0
],
d
.
shape
[
1
],
1
))
data
[:,
:,
0
]
=
d
...
...
@@ -70,7 +77,7 @@ class Disk(Base):
return
data
def
get_batch
(
self
):
def
get_batch
(
self
,
noise
=
False
):
# Shuffling samples
indexes
=
numpy
.
array
(
range
(
self
.
data
.
shape
[
0
]))
...
...
@@ -83,6 +90,8 @@ class Disk(Base):
data
=
self
.
load_from_file
(
file_name
)
selected_data
[
i
,
...]
=
data
# Scaling
if
self
.
scale
:
selected_data
[
i
,
...]
*=
self
.
scale_value
...
...
bob/learn/tensorflow/datashuffler/ImageAugmentation.py
0 → 100644
View file @
109eb4c3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Wed 11 May 2016 09:39:36 CEST
import
bob.ip.base
import
numpy
from
.DataAugmentation
import
DataAugmentation
class
ImageAugmentation
(
DataAugmentation
):
"""
Class for applying common real-time random data augmentation for images.
"""
def
__init__
(
self
,
seed
=
10
):
super
(
ImageAugmentation
,
self
).
__init__
(
seed
=
seed
)
self
.
filter_bank
=
[
self
.
__add_none
,
self
.
__add_none
,
self
.
__add_gaussian_blur
,
self
.
__add_left_right_flip
,
self
.
__add_none
,
self
.
__add_salt_and_pepper
]
#self.__add_rotation,
def
__add_none
(
self
,
image
):
return
image
def
__add_gaussian_blur
(
self
,
image
):
possible_sigmas
=
numpy
.
arange
(
0.1
,
3.
,
0.1
)
possible_radii
=
[
1
,
2
,
3
]
sigma
=
possible_sigmas
[
numpy
.
random
.
randint
(
len
(
possible_sigmas
))]
radius
=
possible_radii
[
numpy
.
random
.
randint
(
len
(
possible_radii
))]
gaussian_filter
=
bob
.
ip
.
base
.
Gaussian
(
sigma
=
(
sigma
,
sigma
),
radius
=
(
radius
,
radius
))
return
gaussian_filter
(
image
)
def
__add_left_right_flip
(
self
,
image
):
return
bob
.
ip
.
base
.
flop
(
image
)
def
__add_rotation
(
self
,
image
):
possible_angles
=
numpy
.
arange
(
-
15
,
15
,
0.5
)
angle
=
possible_angles
[
numpy
.
random
.
randint
(
len
(
possible_angles
))]
return
bob
.
ip
.
base
.
rotate
(
image
,
angle
)
def
__add_salt_and_pepper
(
self
,
image
):
possible_levels
=
numpy
.
arange
(
0.01
,
0.1
,
0.01
)
level
=
possible_levels
[
numpy
.
random
.
randint
(
len
(
possible_levels
))]
return
self
.
compute_salt_and_peper
(
image
,
level
)
def
compute_salt_and_peper
(
self
,
image
,
level
):
"""
Compute a salt and pepper noise
"""
r
=
numpy
.
random
.
rand
(
*
image
.
shape
)
# 0 noise
indexes_0
=
r
<=
(
level
/
0.5
)
image
[
indexes_0
]
=
0.0
# 255 noise
indexes_255
=
(
1
-
level
/
2
)
<=
r
;
image
[
indexes_255
]
=
255.0
return
image
bob/learn/tensorflow/datashuffler/Memory.py
View file @
109eb4c3
...
...
@@ -15,7 +15,8 @@ class Memory(Base):
input_dtype
=
"float64"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
This datashuffler deal with databases that are stored in a :py:class`numpy.array`
...
...
@@ -35,14 +36,12 @@ class Memory(Base):
input_dtype
=
input_dtype
,
scale
=
scale
,
batch_size
=
batch_size
,
seed
=
seed
seed
=
seed
,
data_augmentation
=
data_augmentation
)
# Seting the seed
numpy
.
random
.
seed
(
seed
)
self
.
data
=
self
.
data
.
astype
(
input_dtype
)
if
self
.
scale
:
self
.
data
*=
self
.
scale_value
def
get_batch
(
self
):
...
...
@@ -53,4 +52,14 @@ class Memory(Base):
selected_data
=
self
.
data
[
indexes
[
0
:
self
.
batch_size
],
:,
:,
:]
selected_labels
=
self
.
labels
[
indexes
[
0
:
self
.
batch_size
]]
# Applying the data augmentation
if
self
.
data_augmentation
is
not
None
:
for
i
in
range
(
selected_data
.
shape
[
0
]):
img
=
self
.
skimage2bob
(
selected_data
[
i
,
...])
img
=
self
.
data_augmentation
(
img
)
selected_data
[
i
,
...]
=
self
.
bob2skimage
(
img
)
if
self
.
scale
:
selected_data
*=
self
.
scale_value
return
[
selected_data
.
astype
(
"float32"
),
selected_labels
.
astype
(
"int64"
)]
bob/learn/tensorflow/datashuffler/SiameseDisk.py
View file @
109eb4c3
...
...
@@ -17,7 +17,8 @@ class SiameseDisk(Siamese, Disk):
input_dtype
=
"float64"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
Shuffler that deal with file list
...
...
@@ -43,7 +44,8 @@ class SiameseDisk(Siamese, Disk):
input_dtype
=
input_dtype
,
scale
=
scale
,
batch_size
=
batch_size
,
seed
=
seed
seed
=
seed
,
data_augmentation
=
data_augmentation
)
# Seting the seed
numpy
.
random
.
seed
(
seed
)
...
...
@@ -74,6 +76,7 @@ class SiameseDisk(Siamese, Disk):
labels_siamese
[
i
]
=
not
genuine
genuine
=
not
genuine
if
self
.
scale
:
data
*=
self
.
scale_value
data_p
*=
self
.
scale_value
...
...
bob/learn/tensorflow/datashuffler/SiameseMemory.py
View file @
109eb4c3
...
...
@@ -14,10 +14,11 @@ class SiameseMemory(Siamese, Memory):
def
__init__
(
self
,
data
,
labels
,
input_shape
,
input_dtype
=
"float
64
"
,
input_dtype
=
"float"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
Shuffler that deal with memory datasets
...
...
@@ -37,14 +38,13 @@ class SiameseMemory(Siamese, Memory):
input_dtype
=
input_dtype
,
scale
=
scale
,
batch_size
=
batch_size
,
seed
=
seed
seed
=
seed
,
data_augmentation
=
data_augmentation
)
# Seting the seed
numpy
.
random
.
seed
(
seed
)
self
.
data
=
self
.
data
.
astype
(
input_dtype
)
if
self
.
scale
:
self
.
data
*=
self
.
scale_value
def
get_batch
(
self
,
zero_one_labels
=
True
):
"""
...
...
@@ -55,9 +55,9 @@ class SiameseMemory(Siamese, Memory):
**Return**
"""
data
=
numpy
.
zeros
(
shape
=
self
.
shape
,
dtype
=
'float
32
'
)
data_p
=
numpy
.
zeros
(
shape
=
self
.
shape
,
dtype
=
'float
32
'
)
labels_siamese
=
numpy
.
zeros
(
shape
=
self
.
shape
[
0
],
dtype
=
'float
32
'
)
data
=
numpy
.
zeros
(
shape
=
self
.
shape
,
dtype
=
'float'
)
data_p
=
numpy
.
zeros
(
shape
=
self
.
shape
,
dtype
=
'float'
)
labels_siamese
=
numpy
.
zeros
(
shape
=
self
.
shape
[
0
],
dtype
=
'float'
)
genuine
=
True
for
i
in
range
(
self
.
shape
[
0
]):
...
...
@@ -68,4 +68,18 @@ class SiameseMemory(Siamese, Memory):
labels_siamese
[
i
]
=
-
1
if
genuine
else
+
1
genuine
=
not
genuine
return
[
data
,
data_p
,
labels_siamese
]
# Applying the data augmentation
if
self
.
data_augmentation
is
not
None
:
for
i
in
range
(
data
.
shape
[
0
]):
d
=
self
.
bob2skimage
(
self
.
data_augmentation
(
self
.
skimage2bob
(
data
[
i
,
...])))
data
[
i
,
...]
=
d
d
=
self
.
bob2skimage
(
self
.
data_augmentation
(
self
.
skimage2bob
(
data_p
[
i
,
...])))
data_p
[
i
,
...]
=
d
# Scaling
if
self
.
scale
:
data
*=
self
.
scale_value
data_p
*=
self
.
scale_value
return
[
data
.
astype
(
"float32"
),
data_p
.
astype
(
"float32"
),
labels_siamese
]
bob/learn/tensorflow/datashuffler/TripletDisk.py
View file @
109eb4c3
...
...
@@ -22,7 +22,8 @@ class TripletDisk(Triplet, Disk):
input_dtype
=
"float64"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
Shuffler that deal with file list
...
...
@@ -47,7 +48,8 @@ class TripletDisk(Triplet, Disk):
input_shape
=
input_shape
,
input_dtype
=
input_dtype
,
scale
=
scale
,
batch_size
=
batch_size
batch_size
=
batch_size
,
data_augmentation
=
data_augmentation
)
# Seting the seed
numpy
.
random
.
seed
(
seed
)
...
...
bob/learn/tensorflow/datashuffler/TripletMemory.py
View file @
109eb4c3
...
...
@@ -17,7 +17,8 @@ class TripletMemory(Triplet, Memory):
input_dtype
=
"float64"
,
scale
=
True
,
batch_size
=
1
,
seed
=
10
):
seed
=
10
,
data_augmentation
=
None
):
"""
Shuffler that deal with memory datasets
...
...
@@ -37,14 +38,13 @@ class TripletMemory(Triplet, Memory):
input_dtype
=
input_dtype
,
scale
=
scale
,
batch_size
=
batch_size
,
seed
=
seed
seed
=
seed
,
data_augmentation
=
data_augmentation
)
# Seting the seed
numpy
.
random
.
seed
(
seed
)
self
.
data
=
self
.
data
.
astype
(
input_dtype
)
if
self
.
scale
:
self
.
data
*=
self
.
scale_value
def
get_batch
(
self
):
"""
...
...
@@ -63,4 +63,22 @@ class TripletMemory(Triplet, Memory):
for
i
in
range
(
self
.
shape
[
0
]):
data_a
[
i
,
...],
data_p
[
i
,
...],
data_n
[
i
,
...]
=
self
.
get_one_triplet
(
self
.
data
,
self
.
labels
)
return
data_a
,
data_p
,
data_n
# Applying the data augmentation
if
self
.
data_augmentation
is
not
None
:
for
i
in
range
(
data_a
.
shape
[
0
]):
d
=
self
.
bob2skimage
(
self
.
data_augmentation
(
self
.
skimage2bob
(
data_a
[
i
,
...])))
data_a
[
i
,
...]
=
d
d
=
self
.
bob2skimage
(
self
.
data_augmentation
(
self
.
skimage2bob
(
data_p
[
i
,
...])))
data_p
[
i
,
...]
=
d
d
=
self
.
bob2skimage
(
self
.
data_augmentation
(
self
.
skimage2bob
(
data_n
[
i
,
...])))
data_n
[
i
,
...]
=
d
# Scaling
if
self
.
scale
:
data_a
*=
self
.
scale_value
data_p
*=
self
.
scale_value
data_n
*=
self
.
scale_value
return
[
data_a
.
astype
(
"float32"
),
data_p
.
astype
(
"float32"
),
data_n
.
astype
(
"float32"
)]
bob/learn/tensorflow/datashuffler/__init__.py
View file @
109eb4c3
...
...
@@ -13,5 +13,10 @@ from .TripletMemory import TripletMemory
from
.SiameseDisk
import
SiameseDisk
from
.TripletDisk
import
TripletDisk
# Data Augmentation
from
.DataAugmentation
import
DataAugmentation
from
.ImageAugmentation
import
ImageAugmentation
# gets sphinx autodoc done right - don't remove it
__all__
=
[
_
for
_
in
dir
()
if
not
_
.
startswith
(
'_'
)]
bob/learn/tensorflow/test/test_cnn.py
View file @
109eb4c3
...
...
@@ -4,8 +4,8 @@
# @date: Thu 13 Oct 2016 13:35 CEST
import
numpy
from
bob.learn.tensorflow.datashuffler
import
Memory
,
SiameseMemory
,
TripletMemory
,
Disk
,
SiameseDisk
,
TripletDisk
from
bob.learn.tensorflow.network
import
Chopra
,
MLP
from
bob.learn.tensorflow.datashuffler
import
Memory
,
SiameseMemory
,
TripletMemory
,
Disk
,
SiameseDisk
,
TripletDisk
,
ImageAugmentation
from
bob.learn.tensorflow.network
import
Chopra
,
Lenet
from
bob.learn.tensorflow.loss
import
BaseLoss
,
ContrastiveLoss
,
TripletLoss
from
bob.learn.tensorflow.trainers
import
Trainer
,
SiameseTrainer
,
TripletTrainer
# from ..analyzers import ExperimentAnalizer, SoftmaxAnalizer
...
...
@@ -79,9 +79,11 @@ def test_cnn_trainer():
validation_data
=
numpy
.
reshape
(
validation_data
,
(
validation_data
.
shape
[
0
],
28
,
28
,
1
))
# Creating datashufflers
data_augmentation
=
ImageAugmentation
()
train_data_shuffler
=
Memory
(
train_data
,
train_labels
,
input_shape
=
[
28
,
28
,
1
],
batch_size
=
batch_size
)
batch_size
=
batch_size
,
data_augmentation
=
data_augmentation
)
with
tf
.
Session
()
as
session
:
directory
=
"./temp/cnn"
...
...
@@ -117,6 +119,7 @@ def test_cnn_trainer():
# At least 80% of accuracy
assert
accuracy
>
80.
shutil
.
rmtree
(
directory
)
session
.
close
()
def
test_siamesecnn_trainer
():
...
...
@@ -162,6 +165,7 @@ def test_siamesecnn_trainer():
# At least 80% of accuracy
assert
eer
<
0.25
shutil
.
rmtree
(
directory
)
session
.
close
()
def
test_tripletcnn_trainer
():
...
...
@@ -207,3 +211,4 @@ def test_tripletcnn_trainer():
# At least 80% of accuracy
assert
eer
<
0.25
shutil
.
rmtree
(
directory
)
session
.
close
()
bob/learn/tensorflow/test/test_datashuffler_augmentation.py
0 → 100644
View file @
109eb4c3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# @author: Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
# @date: Thu 13 Oct 2016 13:35 CEST
import
numpy
from
bob.learn.tensorflow.datashuffler
import
Memory
,
SiameseMemory
,
TripletMemory
,
Disk
,
SiameseDisk
,
TripletDisk
,
ImageAugmentation
import
pkg_resources
from
..util
import
load_mnist
import
os
"""
Some unit tests for the datashuffler
"""
def
get_dummy_files
():
base_path
=
pkg_resources
.
resource_filename
(
__name__
,
'data/dummy_database'
)
files
=
[]
clients
=
[]
for
f
in
os
.
listdir
(
base_path
):
if
f
.
endswith
(
".hdf5"
):
files
.
append
(
os
.
path
.
join
(
base_path
,
f
))
clients
.
append
(
int
(
f
[
1
:
4
]))
return
files
,
clients
def
test_memory_shuffler
():
train_data
,
train_labels
,
validation_data
,
validation_labels
=
load_mnist
()
train_data
=
numpy
.
reshape
(
train_data
,
(
train_data
.
shape
[
0
],
28
,
28
,
1
))
batch_shape
=
[
16
,
28
,
28
,
1
]
data_augmentation
=
ImageAugmentation
()
data_shuffler
=
Memory
(
train_data
,
train_labels
,
input_shape
=
batch_shape
[
1
:],
scale
=
True
,
batch_size
=
batch_shape
[
0
],
data_augmentation
=
data_augmentation
)
batch
=
data_shuffler
.
get_batch
()
assert
len
(
batch
)
==
2
assert
batch
[
0
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
1
].
shape
[
0
]
==
batch_shape
[
0
]
def
test_siamesememory_shuffler
():
train_data
,
train_labels
,
validation_data
,
validation_labels
=
load_mnist
()
train_data
=
numpy
.
reshape
(
train_data
,
(
train_data
.
shape
[
0
],
28
,
28
,
1
))
batch_shape
=
[
16
,
28
,
28
,
1
]
data_augmentation
=
ImageAugmentation
()
data_shuffler
=
SiameseMemory
(
train_data
,
train_labels
,
input_shape
=
batch_shape
[
1
:],
scale
=
True
,
batch_size
=
batch_shape
[
0
],
data_augmentation
=
data_augmentation
)
batch
=
data_shuffler
.
get_batch
()
assert
len
(
batch
)
==
3
assert
batch
[
0
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
1
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
2
].
shape
[
0
]
==
batch_shape
[
0
]
placeholders
=
data_shuffler
.
get_placeholders
(
name
=
"train"
)
assert
placeholders
[
0
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
1
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
2
].
get_shape
().
as_list
()[
0
]
==
batch_shape
[
0
]
def
test_tripletmemory_shuffler
():
train_data
,
train_labels
,
validation_data
,
validation_labels
=
load_mnist
()
train_data
=
numpy
.
reshape
(
train_data
,
(
train_data
.
shape
[
0
],
28
,
28
,
1
))
batch_shape
=
[
16
,
28
,
28
,
1
]
data_augmentation
=
ImageAugmentation
()
data_shuffler
=
TripletMemory
(
train_data
,
train_labels
,
input_shape
=
batch_shape
[
1
:],
scale
=
True
,
batch_size
=
batch_shape
[
0
],
data_augmentation
=
data_augmentation
)
batch
=
data_shuffler
.
get_batch
()
assert
len
(
batch
)
==
3
assert
batch
[
0
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
1
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
2
].
shape
==
tuple
(
batch_shape
)
placeholders
=
data_shuffler
.
get_placeholders
(
name
=
"train"
)
assert
placeholders
[
0
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
1
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
2
].
get_shape
().
as_list
()
==
batch_shape
def
test_disk_shuffler
():
train_data
,
train_labels
=
get_dummy_files
()
batch_shape
=
[
2
,
125
,
125
,
3
]
data_augmentation
=
ImageAugmentation
()
data_shuffler
=
Disk
(
train_data
,
train_labels
,
input_shape
=
batch_shape
[
1
:],
scale
=
True
,
batch_size
=
batch_shape
[
0
],
data_augmentation
=
data_augmentation
)
batch
=
data_shuffler
.
get_batch
()
assert
len
(
batch
)
==
2
assert
batch
[
0
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
1
].
shape
[
0
]
==
batch_shape
[
0
]
placeholders
=
data_shuffler
.
get_placeholders
(
name
=
"train"
)
assert
placeholders
[
0
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
1
].
get_shape
().
as_list
()[
0
]
==
batch_shape
[
0
]
def
test_siamesedisk_shuffler
():
train_data
,
train_labels
=
get_dummy_files
()
batch_shape
=
[
2
,
125
,
125
,
3
]
data_augmentation
=
ImageAugmentation
()
data_shuffler
=
SiameseDisk
(
train_data
,
train_labels
,
input_shape
=
batch_shape
[
1
:],
scale
=
True
,
batch_size
=
batch_shape
[
0
],
data_augmentation
=
data_augmentation
)
batch
=
data_shuffler
.
get_batch
()
assert
len
(
batch
)
==
3
assert
batch
[
0
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
1
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
2
].
shape
[
0
]
==
batch_shape
[
0
]
placeholders
=
data_shuffler
.
get_placeholders
(
name
=
"train"
)
assert
placeholders
[
0
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
1
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
2
].
get_shape
().
as_list
()[
0
]
==
batch_shape
[
0
]
def
test_tripletdisk_shuffler
():
train_data
,
train_labels
=
get_dummy_files
()
batch_shape
=
[
1
,
125
,
125
,
3
]
data_augmentation
=
ImageAugmentation
()
data_shuffler
=
TripletDisk
(
train_data
,
train_labels
,
input_shape
=
batch_shape
[
1
:],
scale
=
True
,
batch_size
=
batch_shape
[
0
],
data_augmentation
=
data_augmentation
)
batch
=
data_shuffler
.
get_batch
()
assert
len
(
batch
)
==
3
assert
batch
[
0
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
1
].
shape
==
tuple
(
batch_shape
)
assert
batch
[
2
].
shape
==
tuple
(
batch_shape
)
placeholders
=
data_shuffler
.
get_placeholders
(
name
=
"train"
)
assert
placeholders
[
0
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
1
].
get_shape
().
as_list
()
==
batch_shape
assert
placeholders
[
2
].
get_shape
().
as_list
()
==
batch_shape
bob/learn/tensorflow/test/test_dnn.py
View file @
109eb4c3
...
...
@@ -71,3 +71,4 @@ def test_dnn_trainer():
# At least 50% of accuracy for the DNN
assert
accuracy
>
50.
shutil
.
rmtree
(
directory
)
session
.
close
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment