Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
bob.learn.tensorflow
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
11
Issues
11
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
bob
bob.learn.tensorflow
Commits
ecd82cb4
Commit
ecd82cb4
authored
Oct 31, 2016
by
Tiago de Freitas Pereira
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Preparing batch normalization
parent
54941120
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
75 additions
and
57 deletions
+75
-57
bob/learn/tensorflow/layers/Conv2D.py
bob/learn/tensorflow/layers/Conv2D.py
+1
-1
bob/learn/tensorflow/layers/Layer.py
bob/learn/tensorflow/layers/Layer.py
+29
-22
bob/learn/tensorflow/network/SequenceNetwork.py
bob/learn/tensorflow/network/SequenceNetwork.py
+18
-22
bob/learn/tensorflow/script/train_mobio.py
bob/learn/tensorflow/script/train_mobio.py
+5
-5
bob/learn/tensorflow/test/test_cnn_scratch.py
bob/learn/tensorflow/test/test_cnn_scratch.py
+11
-5
bob/learn/tensorflow/trainers/Trainer.py
bob/learn/tensorflow/trainers/Trainer.py
+11
-2
No files found.
bob/learn/tensorflow/layers/Conv2D.py
View file @
ecd82cb4
...
...
@@ -69,7 +69,7 @@ class Conv2D(Layer):
self
.
b
=
self
.
bias_initialization
(
shape
=
[
self
.
filters
],
name
=
"b_"
+
str
(
self
.
name
)
+
"bias"
,
scope
=
"b_"
+
str
(
self
.
name
)
)
)
def
get_graph
(
self
,
training_phase
=
True
):
...
...
bob/learn/tensorflow/layers/Layer.py
View file @
ecd82cb4
...
...
@@ -40,6 +40,10 @@ class Layer(object):
self
.
input_layer
=
None
self
.
activation
=
activation
# Batch normalization variables
self
.
beta
=
None
self
.
gamma
=
None
def
create_variables
(
self
,
input_layer
):
NotImplementedError
(
"Please implement this function in derived classes"
)
...
...
@@ -61,30 +65,33 @@ class Layer(object):
"""
from
tensorflow.python.ops
import
control_flow_ops
name
=
"batch_norm"
with
tf
.
variable_scope
(
name
):
phase_train
=
tf
.
convert_to_tensor
(
phase_train
,
dtype
=
tf
.
bool
)
n_out
=
int
(
x
.
get_shape
()[
-
1
])
beta
=
tf
.
Variable
(
tf
.
constant
(
0.0
,
shape
=
[
n_out
],
dtype
=
x
.
dtype
),
name
=
name
+
'/beta'
,
trainable
=
True
,
dtype
=
x
.
dtype
)
gamma
=
tf
.
Variable
(
tf
.
constant
(
1.0
,
shape
=
[
n_out
],
dtype
=
x
.
dtype
),
name
=
name
+
'/gamma'
,
trainable
=
True
,
dtype
=
x
.
dtype
)
# If signal
#if len(x.get_shape()) == 2:
# batch_mean, batch_var = tf.nn.moments(x, [0], name='moments_{0}'.format(name))
#else:
name
=
"batch_norm_"
+
str
(
self
.
name
)
#with tf.variable_scope(name):
phase_train
=
tf
.
convert_to_tensor
(
phase_train
,
dtype
=
tf
.
bool
)
n_out
=
int
(
x
.
get_shape
()[
-
1
])
self
.
beta
=
tf
.
Variable
(
tf
.
constant
(
0.0
,
shape
=
[
n_out
],
dtype
=
x
.
dtype
),
name
=
name
+
'_beta'
,
trainable
=
True
,
dtype
=
x
.
dtype
)
self
.
gamma
=
tf
.
Variable
(
tf
.
constant
(
1.0
,
shape
=
[
n_out
],
dtype
=
x
.
dtype
),
name
=
name
+
'_gamma'
,
trainable
=
True
,
dtype
=
x
.
dtype
)
if
len
(
x
.
get_shape
())
==
2
:
batch_mean
,
batch_var
=
tf
.
nn
.
moments
(
x
,
[
0
],
name
=
'moments_{0}'
.
format
(
name
))
else
:
batch_mean
,
batch_var
=
tf
.
nn
.
moments
(
x
,
range
(
len
(
x
.
get_shape
())
-
1
),
name
=
'moments_{0}'
.
format
(
name
))
ema
=
tf
.
train
.
ExponentialMovingAverage
(
decay
=
0.9
)
ema
=
tf
.
train
.
ExponentialMovingAverage
(
decay
=
0.9
)
def
mean_var_with_update
():
ema_apply_op
=
ema
.
apply
([
batch_mean
,
batch_var
])
with
tf
.
control_dependencies
([
ema_apply_op
]):
return
tf
.
identity
(
batch_mean
),
tf
.
identity
(
batch_var
)
def
mean_var_with_update
():
ema_apply_op
=
ema
.
apply
([
batch_mean
,
batch_var
])
with
tf
.
control_dependencies
([
ema_apply_op
]):
return
tf
.
identity
(
batch_mean
),
tf
.
identity
(
batch_var
)
mean
,
var
=
control_flow_ops
.
cond
(
phase_train
,
mean_var_with_update
,
lambda
:
(
ema
.
average
(
batch_mean
),
ema
.
average
(
batch_var
)))
normed
=
tf
.
nn
.
batch_normalization
(
x
,
mean
,
var
,
beta
,
gamma
,
1e-3
)
mean
,
var
=
control_flow_ops
.
cond
(
phase_train
,
mean_var_with_update
,
lambda
:
(
ema
.
average
(
batch_mean
),
ema
.
average
(
batch_var
)))
normed
=
tf
.
nn
.
batch_normalization
(
x
,
mean
,
var
,
self
.
beta
,
self
.
gamma
,
1e-3
)
return
normed
bob/learn/tensorflow/network/SequenceNetwork.py
View file @
ecd82cb4
...
...
@@ -125,6 +125,11 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
variables
[
self
.
sequence_net
[
k
].
W
.
name
]
=
self
.
sequence_net
[
k
].
W
variables
[
self
.
sequence_net
[
k
].
b
.
name
]
=
self
.
sequence_net
[
k
].
b
# Dumping batch norm variables
if
self
.
sequence_net
[
k
].
batch_norm
:
variables
[
self
.
sequence_net
[
k
].
beta
.
name
]
=
self
.
sequence_net
[
k
].
beta
variables
[
self
.
sequence_net
[
k
].
gamma
.
name
]
=
self
.
sequence_net
[
k
].
gamma
return
variables
def
variable_summaries
(
self
,
var
,
name
):
...
...
@@ -200,7 +205,8 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
split_path
=
path
.
split
(
"/"
)
for
i
in
range
(
0
,
len
(
split_path
)
-
1
):
p
=
split_path
[
i
]
hdf5
.
create_group
(
p
)
if
not
hdf5
.
has_group
(
p
):
hdf5
.
create_group
(
p
)
# Saving the architecture
if
self
.
pickle_architecture
is
not
None
:
...
...
@@ -237,6 +243,9 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
session
.
run
(
self
.
sequence_net
[
k
].
W
)
self
.
sequence_net
[
k
].
b
.
assign
(
hdf5
.
read
(
self
.
sequence_net
[
k
].
b
.
name
)).
eval
(
session
=
session
)
session
.
run
(
self
.
sequence_net
[
k
].
b
)
hdf5
.
cd
(
".."
)
def
load
(
self
,
hdf5
,
shape
=
None
,
session
=
None
,
batch
=
1
,
use_gpu
=
False
):
...
...
@@ -276,26 +285,15 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
tf
.
initialize_all_variables
().
run
(
session
=
session
)
self
.
load_variables_only
(
hdf5
,
session
)
"""
def save(self, session, path, step=None):
if self.saver is None:
#variables = self.dump_variables()
#variables['mean'] = tf.Variable(10.0)
#import ipdb; ipdb.set_trace()
def
save_original
(
self
,
session
,
saver
,
path
):
return
saver
.
save
(
session
,
path
)
#tf.initialize_all_variables().run()
self.saver = tf.train.Saver(session)
def
load_original
(
self
,
session
,
path
):
saver
=
tf
.
train
.
import_meta_graph
(
path
+
".meta"
)
saver
.
restore
(
session
,
path
)
if step is None:
return self.saver.save(session, path)
else:
return self.saver.save(session, path)
def load(self, path, session=None):
if session is None:
session = tf.Session()
#if session is None:
# session = tf.Session()
#tf.initialize_all_variables().run(session=session)
# Loading variables
...
...
@@ -308,6 +306,4 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
#variables['input_divide'] = self.input_divide
#variables['input_subtract'] = self.input_subtract
#self.saver = tf.train.Saver(variables)
self.saver.restore(session, path)
"""
\ No newline at end of file
#self.saver.restore(session, path)
bob/learn/tensorflow/script/train_mobio.py
View file @
ecd82cb4
...
...
@@ -29,8 +29,8 @@ from bob.learn.tensorflow.loss import ContrastiveLoss, BaseLoss, TripletLoss
import
numpy
import
os
os
.
environ
[
"CUDA_VISIBLE_DEVICES"
]
=
"0,1,2,3"
#
os.environ["CUDA_VISIBLE_DEVICES"] = ""
#
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
os
.
environ
[
"CUDA_VISIBLE_DEVICES"
]
=
""
def
main
():
...
...
@@ -45,8 +45,8 @@ def main():
import
bob.db.mobio
db_mobio
=
bob
.
db
.
mobio
.
Database
()
#
directory = "/idiap/temp/tpereira/DEEP_FACE/CASIA_WEBFACE/mobio/preprocessed/"
directory
=
"./preprocessed/"
directory
=
"/idiap/temp/tpereira/DEEP_FACE/CASIA_WEBFACE/mobio/preprocessed/"
#
directory = "./preprocessed/"
# Preparing train set
#train_objects = db_mobio.objects(protocol="male", groups="world")
...
...
@@ -108,7 +108,7 @@ def main():
prefetch
=
False
,
optimizer
=
optimizer
,
snapshot
=
200
,
temp_dir
=
"
./logs/
"
)
temp_dir
=
"
/idiap/temp/tpereira/VGG16/mobio_world
"
)
#trainer.train(train_data_shuffler, validation_data_shuffler)
trainer
.
train
(
train_data_shuffler
)
bob/learn/tensorflow/test/test_cnn_scratch.py
View file @
ecd82cb4
...
...
@@ -28,13 +28,13 @@ seed = 10
def
scratch_network
():
# Creating a random network
scratch
=
SequenceNetwork
()
scratch
=
SequenceNetwork
(
default_feature_layer
=
"fc1"
)
scratch
.
add
(
Conv2D
(
name
=
"conv1"
,
kernel_size
=
3
,
filters
=
10
,
activation
=
tf
.
nn
.
tanh
,
weights_initialization
=
Xavier
(
seed
=
seed
,
use_gpu
=
False
),
bias_initialization
=
Constant
(
use_gpu
=
False
)
))
bias_initialization
=
Constant
(
use_gpu
=
False
)
,
batch_norm
=
True
))
scratch
.
add
(
FullyConnected
(
name
=
"fc1"
,
output_dim
=
10
,
activation
=
None
,
weights_initialization
=
Xavier
(
seed
=
seed
,
use_gpu
=
False
),
...
...
@@ -50,11 +50,15 @@ def validate_network(validation_data, validation_labels, directory):
input_shape
=
[
28
,
28
,
1
],
batch_size
=
validation_batch_size
)
with
tf
.
Session
()
as
session
:
validation_shape
=
[
400
,
28
,
28
,
1
]
path
=
os
.
path
.
join
(
directory
,
"model.hdf5"
)
#path = os.path.join(directory, "model.ckp")
#scratch = SequenceNetwork(default_feature_layer="fc1")
scratch
=
SequenceNetwork
()
scratch
.
load
(
bob
.
io
.
base
.
HDF5File
(
os
.
path
.
join
(
directory
,
"model.hdf5"
)),
#scratch.load_original(session, os.path.join(directory, "model.ckp"))
scratch
.
load
(
bob
.
io
.
base
.
HDF5File
(
path
),
shape
=
validation_shape
,
session
=
session
)
[
data
,
labels
]
=
validation_data_shuffler
.
get_batch
()
predictions
=
scratch
(
data
,
session
=
session
)
accuracy
=
100.
*
numpy
.
sum
(
numpy
.
argmax
(
predictions
,
1
)
==
labels
)
/
predictions
.
shape
[
0
]
...
...
@@ -91,6 +95,8 @@ def test_cnn_trainer_scratch():
temp_dir
=
directory
)
trainer
.
train
(
train_data_shuffler
)
import
ipdb
;
ipdb
.
set_trace
();
accuracy
=
validate_network
(
validation_data
,
validation_labels
,
directory
)
assert
accuracy
>
80
...
...
bob/learn/tensorflow/trainers/Trainer.py
View file @
ecd82cb4
...
...
@@ -292,6 +292,9 @@ class Trainer(object):
with
tf
.
Session
(
config
=
config
)
as
session
:
tf
.
initialize_all_variables
().
run
()
# Original tensorflow saver object
saver
=
tf
.
train
.
Saver
(
var_list
=
tf
.
trainable_variables
())
# Loading a pretrained model
if
self
.
model_from_file
!=
""
:
logger
.
info
(
"Loading pretrained model from {0}"
.
format
(
self
.
model_from_file
))
...
...
@@ -328,7 +331,10 @@ class Trainer(object):
# Taking snapshot
if
step
%
self
.
snapshot
==
0
:
logger
.
info
(
"Taking snapshot"
)
hdf5
=
bob
.
io
.
base
.
HDF5File
(
os
.
path
.
join
(
self
.
temp_dir
,
'model_snapshot{0}.hdf5'
.
format
(
step
)),
'w'
)
path
=
os
.
path
.
join
(
self
.
temp_dir
,
'model_snapshot{0}.hdf5'
.
format
(
step
))
#path_original = os.path.join(self.temp_dir, 'model_snapshot{0}.ckp'.format(step))
#self.architecture.save_original(session, saver, path_original)
hdf5
=
bob
.
io
.
base
.
HDF5File
(
path
,
'w'
)
self
.
architecture
.
save
(
hdf5
)
del
hdf5
...
...
@@ -339,7 +345,10 @@ class Trainer(object):
self
.
validation_summary_writter
.
close
()
# Saving the final network
hdf5
=
bob
.
io
.
base
.
HDF5File
(
os
.
path
.
join
(
self
.
temp_dir
,
'model.hdf5'
),
'w'
)
path
=
os
.
path
.
join
(
self
.
temp_dir
,
'model.hdf5'
)
#path_original = os.path.join(self.temp_dir, 'model.ckp')
#self.architecture.save_original(session, saver, path_original)
hdf5
=
bob
.
io
.
base
.
HDF5File
(
path
,
'w'
)
self
.
architecture
.
save
(
hdf5
)
del
hdf5
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment