Removed the hdf5 backend and tests with batch normalization

parent f8248a9e
......@@ -18,37 +18,14 @@ class SoftmaxAnalizer(object):
def __init__(self):
"""
Softmax analizer
** Parameters **
data_shuffler:
graph:
session:
convergence_threshold:
convergence_reference: References to analize the convergence. Possible values are `eer`, `far10`, `far10`
"""
self.data_shuffler = None
self.network = None
self.session = None
pass
def __call__(self, data_shuffler, network, session):
data, labels = data_shuffler.get_batch()
if self.data_shuffler is None:
self.data_shuffler = data_shuffler
self.network = network
self.session = session
# Creating the graph
feature_batch, label_batch = self.data_shuffler.get_placeholders(name="validation_accuracy")
data, labels = self.data_shuffler.get_batch()
graph = self.network.compute_graph(feature_batch)
predictions = numpy.argmax(self.session.run(graph, feed_dict={feature_batch: data[:]}), 1)
predictions = numpy.argmax(session.run(network.inference_graph, feed_dict={network.inference_placeholder: data[:]}), 1)
accuracy = 100. * numpy.sum(predictions == labels) / predictions.shape[0]
summaries = []
summaries.append(summary_pb2.Summary.Value(tag="accuracy_validation", simple_value=float(accuracy)))
summaries = [(summary_pb2.Summary.Value(tag="accuracy_validation", simple_value=float(accuracy)))]
return summary_pb2.Summary(value=summaries)
......@@ -58,7 +58,7 @@ class Base(object):
self.label_placeholder = None
self.data_augmentation = data_augmentation
self.deployment_shape = [-1] + list(input_shape)
self.deployment_shape = [None] + list(input_shape)
def get_placeholders(self, name=""):
"""
......
......@@ -5,6 +5,7 @@
import logging
logger = logging.getLogger("bob.learn.tensorflow")
import tensorflow as tf
class Initialization(object):
......@@ -25,6 +26,10 @@ class Initialization(object):
self.seed = seed
self.use_gpu = use_gpu
tf.set_random_seed(seed)
def variable_exist(self, var):
return var in [v.name.split("/")[0] for v in tf.all_variables()]
def __call__(self, shape, name, scope):
NotImplementedError("Please implement this function in derived classes")
......@@ -39,6 +39,18 @@ class Xavier(Initialization):
initializer = tf.truncated_normal(shape, stddev=stddev, seed=self.seed)
reuse = self.variable_exist(scope)
"""
with tf.variable_scope(scope, reuse=reuse):
if self.use_gpu:
with tf.device("/gpu:0"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
else:
with tf.device("/cpu"):
return tf.get_variable(name, initializer=initializer, dtype=tf.float32)
"""
try:
with tf.variable_scope(scope):
if self.use_gpu:
......
......@@ -68,8 +68,7 @@ class Conv2D(Layer):
self.b = self.bias_initialization(shape=[self.filters],
name="b_" + str(self.name) + "bias",
scope="b_" + str(self.name)
)
scope="b_" + str(self.name))
def get_graph(self, training_phase=True):
......
......@@ -43,6 +43,8 @@ class Layer(object):
# Batch normalization variables
self.beta = None
self.gamma = None
self.batch_mean = None
self.batch_var = None
def create_variables(self, input_layer):
NotImplementedError("Please implement this function in derived classes")
......@@ -50,6 +52,9 @@ class Layer(object):
def get_graph(self, training_phase=True):
NotImplementedError("Please implement this function in derived classes")
def variable_exist(self, var):
return var in [v.name.split("/")[0] for v in tf.all_variables()]
def batch_normalize(self, x, phase_train):
"""
Batch normalization on convolutional maps.
......@@ -66,32 +71,42 @@ class Layer(object):
from tensorflow.python.ops import control_flow_ops
name = "batch_norm_" + str(self.name)
#with tf.variable_scope(name):
phase_train = tf.convert_to_tensor(phase_train, dtype=tf.bool)
n_out = int(x.get_shape()[-1])
self.beta = tf.get_variable(name + '_beta',
initializer=tf.constant(0.0, shape=[n_out], dtype=x.dtype),
trainable=True,
dtype=x.dtype)
self.gamma = tf.get_variable(name + '_gamma',
initializer=tf.constant(1.0, shape=[n_out], dtype=x.dtype),
trainable=True,
dtype=x.dtype)
if len(x.get_shape()) == 2:
batch_mean, batch_var = tf.nn.moments(x, [0], name='moments_{0}'.format(name))
else:
batch_mean, batch_var = tf.nn.moments(x, range(len(x.get_shape())-1), name='moments_{0}'.format(name))
ema = tf.train.ExponentialMovingAverage(decay=0.9)
def mean_var_with_update():
ema_apply_op = ema.apply([batch_mean, batch_var])
with tf.control_dependencies([ema_apply_op]):
return tf.identity(batch_mean), tf.identity(batch_var)
mean, var = control_flow_ops.cond(phase_train,
mean_var_with_update,
lambda: (ema.average(batch_mean), ema.average(batch_var)))
normed = tf.nn.batch_normalization(x, mean, var, self.beta, self.gamma, 1e-3)
reuse = self.variable_exist(name)
#if reuse:
#import ipdb; ipdb.set_trace();
with tf.variable_scope(name, reuse=reuse):
phase_train = tf.convert_to_tensor(phase_train, dtype=tf.bool)
n_out = int(x.get_shape()[-1])
self.beta = tf.get_variable(name + '_beta',
initializer=tf.constant(0.0, shape=[n_out], dtype=x.dtype),
trainable=True,
dtype=x.dtype)
self.gamma = tf.get_variable(name + '_gamma',
initializer=tf.constant(1.0, shape=[n_out], dtype=x.dtype),
trainable=True,
dtype=x.dtype)
if len(x.get_shape()) == 2:
self.batch_mean, self.batch_var = tf.nn.moments(x, [0], name='moments_{0}'.format(name))
else:
self.batch_mean, self.batch_var = tf.nn.moments(x, range(len(x.get_shape())-1), name='moments_{0}'.format(name))
ema = tf.train.ExponentialMovingAverage(decay=0.9)
def mean_var_with_update():
ema_apply_op = ema.apply([self.batch_mean, self.batch_var])
with tf.control_dependencies([ema_apply_op]):
return tf.identity(self.batch_mean), tf.identity(self.batch_var)
mean, var = control_flow_ops.cond(phase_train,
mean_var_with_update,
lambda: (ema.average(self.batch_mean), ema.average(self.batch_var)),
name=name + "mean_var")
normed = tf.nn.batch_normalization(x, mean, var, self.beta, self.gamma, 1e-3)
return normed
......@@ -75,28 +75,30 @@ class Chopra(SequenceNetwork):
default_feature_layer="fc1",
seed=10,
use_gpu=False):
use_gpu=False,
batch_norm=False):
super(Chopra, self).__init__(default_feature_layer=default_feature_layer,
use_gpu=use_gpu)
self.add(Conv2D(name="conv1", kernel_size=conv1_kernel_size,
filters=conv1_output,
activation=None,
activation=tf.nn.relu,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)
bias_initialization=Constant(use_gpu=self.use_gpu),
batch_norm=batch_norm
))
self.add(MaxPooling(name="pooling1", shape=pooling1_size, activation=tf.nn.tanh))
self.add(MaxPooling(name="pooling1", shape=pooling1_size, activation=tf.nn.relu, batch_norm=False))
self.add(Conv2D(name="conv2", kernel_size=conv2_kernel_size,
filters=conv2_output,
activation=None,
activation=tf.nn.relu,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)
))
self.add(MaxPooling(name="pooling2", shape=pooling2_size, activation=tf.nn.tanh))
bias_initialization=Constant(use_gpu=self.use_gpu),
batch_norm=batch_norm))
self.add(MaxPooling(name="pooling2", shape=pooling2_size, activation=tf.nn.relu, batch_norm=False))
self.add(FullyConnected(name="fc1", output_dim=fc1_output,
activation=None,
weights_initialization=Xavier(seed=seed, use_gpu=self.use_gpu),
bias_initialization=Constant(use_gpu=self.use_gpu)))
bias_initialization=Constant(use_gpu=self.use_gpu), batch_norm=False))
......@@ -38,6 +38,10 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
self.pickle_architecture = None# The trainer triggers this
self.deployment_shape = None# The trainer triggers this
# Inference graph
self.inference_graph = None
self.inference_placeholder = None
def add(self, layer):
"""
Add a :py:class:`bob.learn.tensorflow.layers.Layer` in the sequence network
......@@ -77,17 +81,23 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
return input_offset
def compute_projection_graph(self, placeholder):
def compute_inference_graph(self, feature_layer=None):
"""Generate a graph for feature extraction
**Parameters**
placeholder: tensorflow placeholder as input data
"""
return self.compute_graph(placeholder)
if feature_layer is None:
feature_layer = self.default_feature_layer
self.inference_graph = self.compute_graph(self.inference_placeholder, feature_layer, training=False)
def compute_inference_placeholder(self, data_shape):
self.inference_placeholder = tf.placeholder(tf.float32, shape=data_shape, name="feature")
def __call__(self, data, session=None, feature_layer=None):
"""Run a graph
"""Run a graph and compute the embeddings
**Parameters**
......@@ -103,16 +113,16 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
session = tf.Session()
# Feeding the placeholder
feature_placeholder = tf.placeholder(tf.float32, shape=data.shape, name="feature")
feed_dict = {feature_placeholder: data}
if self.inference_placeholder is None:
self.compute_inference_placeholder(data.shape[1:])
feed_dict = {self.inference_placeholder: data}
if feature_layer is None:
feature_layer = self.default_feature_layer
if self.inference_graph is None:
self.compute_inference_graph(self.inference_placeholder, feature_layer)
feature = session.run([self.compute_graph(feature_placeholder, feature_layer, training=False)], feed_dict=feed_dict)[0]
del feature_placeholder
embedding = session.run([self.inference_graph], feed_dict=feed_dict)[0]
return feature
return embedding
def dump_variables(self):
"""
......@@ -129,6 +139,8 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
if self.sequence_net[k].batch_norm:
variables[self.sequence_net[k].beta.name] = self.sequence_net[k].beta
variables[self.sequence_net[k].gamma.name] = self.sequence_net[k].gamma
#variables[self.sequence_net[k].mean.name] = self.sequence_net[k].mean
#variables[self.sequence_net[k].var.name] = self.sequence_net[k].var
return variables
......@@ -189,7 +201,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
return samples_per_sample
def save(self, hdf5):
def save_hdf5(self, hdf5):
"""
Save the state of the network in HDF5 format
......@@ -250,7 +262,7 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
hdf5.cd("..")
def load(self, hdf5, shape=None, session=None, batch=1, use_gpu=False):
def load_hdf5(self, hdf5, shape=None, session=None, batch=1, use_gpu=False):
"""
Load the network from scratch.
This will build the graphs
......@@ -287,25 +299,15 @@ class SequenceNetwork(six.with_metaclass(abc.ABCMeta, object)):
tf.initialize_all_variables().run(session=session)
self.load_variables_only(hdf5, session)
def save_original(self, session, saver, path):
def save(self, session, saver, path):
open(path+"_sequence_net.pickle", 'w').write(self.pickle_architecture)
return saver.save(session, path)
def load_original(self, session, path):
def load(self, session, path):
self.sequence_net = pickle.loads(open(path+"_sequence_net.pickle").read())
saver = tf.train.import_meta_graph(path + ".meta")
saver.restore(session, path)
self.inference_graph = tf.get_collection("inference_graph")[0]
self.inference_placeholder = tf.get_collection("inference_placeholder")[0]
#if session is None:
# session = tf.Session()
#tf.initialize_all_variables().run(session=session)
# Loading variables
#place_holder = tf.placeholder(tf.float32, shape=shape, name="load")
#self.compute_graph(place_holder)
#tf.initialize_all_variables().run(session=session)
#if self.saver is None:
#variables = self.dump_variables()
#variables['input_divide'] = self.input_divide
#variables['input_subtract'] = self.input_subtract
#self.saver = tf.train.Saver(variables)
#self.saver.restore(session, path)
return saver
......@@ -59,21 +59,19 @@ def main():
input_shape=[28, 28, 1],
batch_size=VALIDATION_BATCH_SIZE)
# Preparing the architecture
cnn = True
if cnn:
architecture = Chopra(seed=SEED, fc1_output=10)
#architecture = Lenet(seed=SEED)
#architecture = Dummy(seed=SEED)
architecture = Chopra(seed=SEED, fc1_output=10, batch_norm=False)
loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
#trainer = Trainer(architecture=architecture,
# loss=loss,
# iterations=ITERATIONS,
# analizer=ExperimentAnalizer(),
# prefetch=False, temp_dir="./temp/cnn")
#trainer.train(train_data_shuffler, validation_data_shuffler)
trainer = Trainer(architecture=architecture,
loss=loss,
iterations=ITERATIONS,
prefetch=False, temp_dir="./temp/cnn/no-batch-norm-all-relu")
#prefetch = False, temp_dir = "./temp/cnn/batch-norm-2convs-all-relu")
trainer.train(train_data_shuffler, validation_data_shuffler)
#trainer.train(train_data_shuffler)
else:
mlp = MLP(10, hidden_layers=[15, 20])
......@@ -82,16 +80,16 @@ def main():
trainer.train(train_data_shuffler, validation_data_shuffler)
# Loading
test_data_shuffler = Memory(validation_data, validation_labels,
input_shape=[28, 28, 1],
batch_size=400)
#test_data_shuffler = Memory(validation_data, validation_labels,
# input_shape=[28, 28, 1],
# batch_size=400)
with tf.Session() as session:
new_net = Chopra(seed=SEED, fc1_output=10)
new_net.load(bob.io.base.HDF5File("./temp/cnn/model.hdf5"), shape=[400, 28, 28, 1], session=session)
#with tf.Session() as session:
#new_net = Chopra(seed=SEED, fc1_output=10)
#new_net.load(bob.io.base.HDF5File("./temp/cnn/model.hdf5"), shape=[400, 28, 28, 1], session=session)
[data, labels] = test_data_shuffler.get_batch()
print new_net(data, session)
#[data, labels] = test_data_shuffler.get_batch()
#print new_net(data, session)
......
......@@ -23,7 +23,7 @@ import tensorflow as tf
from .. import util
SEED = 10
from bob.learn.tensorflow.datashuffler import TripletDisk, TripletWithSelectionDisk, TripletWithFastSelectionDisk
from bob.learn.tensorflow.network import Lenet, MLP, LenetDropout, VGG, Chopra, Dummy, FaceNet, FaceNetSimple
from bob.learn.tensorflow.network import Lenet, MLP, LenetDropout, VGG, Chopra, Dummy, FaceNet, FaceNetSimple, VGG16
from bob.learn.tensorflow.trainers import SiameseTrainer, TripletTrainer, constant
from bob.learn.tensorflow.loss import ContrastiveLoss, TripletLoss
import numpy
......@@ -56,10 +56,14 @@ def main():
extension=".hdf5")
for o in train_objects]
train_data_shuffler = TripletWithFastSelectionDisk(train_file_names, train_labels,
input_shape=[224, 224, 3],
batch_size=BATCH_SIZE,
total_identities=16)
#train_data_shuffler = TripletWithFastSelectionDisk(train_file_names, train_labels,
# input_shape=[224, 224, 3],
# batch_size=BATCH_SIZE,
# total_identities=16)
train_data_shuffler = TripletDisk(train_file_names, train_labels,
input_shape=[224, 224, 3],
batch_size=VALIDATION_BATCH_SIZE)
# Preparing train set
......@@ -77,7 +81,8 @@ def main():
batch_size=VALIDATION_BATCH_SIZE)
# Preparing the architecture
# LENET PAPER CHOPRA
architecture = FaceNetSimple(seed=SEED, use_gpu=USE_GPU)
#architecture = FaceNetSimple(seed=SEED, use_gpu=USE_GPU)
architecture = VGG16(seed=SEED, use_gpu=USE_GPU)
optimizer = tf.train.GradientDescentOptimizer(0.05)
loss = TripletLoss(margin=0.2)
......
......@@ -86,29 +86,30 @@ def test_cnn_trainer():
batch_size=batch_size,
data_augmentation=data_augmentation)
with tf.Session() as session:
directory = "./temp/cnn"
directory = "./temp/cnn"
# Preparing the architecture
architecture = Chopra(seed=seed, fc1_output=10)
# Preparing the architecture
architecture = Chopra(seed=seed, fc1_output=10)
# Loss for the softmax
loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
# Loss for the softmax
loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean)
# One graph trainer
trainer = Trainer(architecture=architecture,
loss=loss,
iterations=iterations,
analizer=None,
prefetch=False,
temp_dir=directory)
trainer.train(train_data_shuffler)
del trainer #Just to clean tf.variables
# One graph trainer
trainer = Trainer(architecture=architecture,
loss=loss,
iterations=iterations,
analizer=None,
prefetch=False,
temp_dir=directory)
trainer.train(train_data_shuffler)
with tf.Session() as session:
# Testing
validation_shape = [400, 28, 28, 1]
chopra = Chopra(seed=seed, fc1_output=10)
chopra.load(bob.io.base.HDF5File(os.path.join(directory, "model.hdf5")),
shape=validation_shape, session=session)
chopra.load(session, os.path.join(directory, "model.ckp"))
validation_data_shuffler = Memory(validation_data, validation_labels,
input_shape=[28, 28, 1],
batch_size=validation_batch_size)
......@@ -135,33 +136,32 @@ def test_siamesecnn_trainer():
input_shape=[28, 28, 1],
batch_size=validation_batch_size)
with tf.Session() as session:
directory = "./temp/siamesecnn"
directory = "./temp/siamesecnn"
# Preparing the architecture
architecture = Chopra(seed=seed, fc1_output=10)
# Preparing the architecture
architecture = Chopra(seed=seed, fc1_output=10)
# Loss for the Siamese
loss = ContrastiveLoss(contrastive_margin=4.)
# Loss for the Siamese
loss = ContrastiveLoss(contrastive_margin=4.)
# One graph trainer
trainer = SiameseTrainer(architecture=architecture,
loss=loss,
iterations=iterations,
prefetch=False,
analizer=None,
learning_rate=constant(0.05, name="siamese_lr"),
temp_dir=directory)
# One graph trainer
trainer = SiameseTrainer(architecture=architecture,
loss=loss,
iterations=iterations,
prefetch=False,
analizer=None,
learning_rate=constant(0.05, name="siamese_lr"),
temp_dir=directory)
trainer.train(train_data_shuffler)
trainer.train(train_data_shuffler)
del trainer # Just to clean tf.variables
with tf.Session() as session:
# Testing
validation_shape = [400, 28, 28, 1]
chopra = Chopra(seed=seed, fc1_output=10)
chopra.load(bob.io.base.HDF5File(os.path.join(directory, "model.hdf5")),
shape=validation_shape, session=session)
chopra.load(session, os.path.join(directory, "model.ckp"))
eer = dummy_experiment(validation_data_shuffler, architecture, session)
eer = dummy_experiment(validation_data_shuffler, chopra, session)
# At least 80% of accuracy
assert eer < 0.25
......@@ -181,33 +181,33 @@ def test_tripletcnn_trainer():
input_shape=[28, 28, 1],
batch_size=validation_batch_size)
with tf.Session() as session:
directory = "./temp/tripletcnn"
directory = "./temp/tripletcnn"
# Preparing the architecture
architecture = Chopra(seed=seed, fc1_output=10)
# Preparing the architecture
architecture = Chopra(seed=seed, fc1_output=10)
# Loss for the Siamese
loss = TripletLoss(margin=4.)
# Loss for the Siamese
loss = TripletLoss(margin=4.)
# One graph trainer
trainer = TripletTrainer(architecture=architecture,
loss=loss,
iterations=iterations,
prefetch=False,
analizer=None,
learning_rate=constant(0.05, name="triplet_lr"),
temp_dir=directory)
# One graph trainer
trainer = TripletTrainer(architecture=architecture,
loss=loss,
iterations=iterations,
prefetch=False,
analizer=None,
learning_rate=constant(0.05, name="triplet_lr"),
temp_dir=directory)
trainer.train(train_data_shuffler)
trainer.train(train_data_shuffler)
del trainer # Just to clean tf.variables
with tf.Session() as session:
# Testing
validation_shape = [400, 28, 28, 1]
chopra = Chopra(seed=seed, fc1_output=10)
chopra.load(bob.io.base.HDF5File(os.path.join(directory, "model.hdf5")),
shape=validation_shape, session=session)
chopra.load(session, os.path.join(directory, "model.ckp"))
eer = dummy_experiment(validation_data_shuffler, architecture, session)
eer = dummy_experiment(validation_data_shuffler, chopra, session)
# At least 80% of accuracy
assert eer < 0.25
......
......@@ -61,18 +61,20 @@ def test_cnn_trainer_scratch():
del scratch
del loss
del trainer
# Training the network using a pre trained model
loss2 = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean, name="loss2")
loss = BaseLoss(tf.nn.sparse_softmax_cross_entropy_with_logits, tf.reduce_mean, name="loss")
scratch = scratch_network()
trainer2 = Trainer(architecture=scratch,
loss=loss2,
iterations=iterations,
analizer=None,
prefetch=False,
learning_rate=constant(0.05, name="lr2"),
temp_dir=directory2,
model_from_file=os.path.join(directory, "model.hdf5"))
trainer = Trainer(architecture=scratch,
loss=loss,
iterations=iterations,
analizer=None,
prefetch=False,
learning_rate=constant(0.05, name="lr2"),
temp_dir=directory2,
model_from_file=os.path.join(directory, "model.ckp"))
trainer2.train(train_data_shuffler)
trainer.train(train_data_shuffler)
accuracy = validate_network(validation_data, validation_labels, directory)
assert accuracy > 90
......@@ -24,6 +24,7 @@ batch_size = 16
validation_batch_size = 400
iterations = 50
seed = 10
directory = "./temp/cnn_scratch"
def scratch_network():
......@@ -34,11 +35,12 @@ def scratch_network():
activation=tf.nn.tanh,
weights_initialization=Xavier(seed=seed, use_gpu=False),
bias_initialization=Constant(use_gpu=False),
batch_norm=True))
batch_norm=False))
scratch.add(FullyConnected(name="fc1", output_dim=10,
activation=None,
weights_initialization=Xavier(seed=seed, use_gpu=False),
bias_initialization=Constant(use_gpu=False)
bias_initialization=Constant(use_gpu=False),
batch_norm=False
))
return scratch
......@@ -50,15 +52,8 @@ def validate_network(validation_data, validation_labels, directory):
input_shape=[28, 28, 1],
batch_size=validation_batch_size)
with tf.Session() as session:
validation_shape = [400, 28, 28, 1]
path = os.path.join(directory, "model.hdf5")
#path = os.path.join(directory, "model.ckp")
#scratch = SequenceNetwork(default_feature_layer="fc1")
scratch = SequenceNetwork(default_feature_layer="fc1")
#scratch.load_original(session, os.path.join(directory, "model.ckp"))
scratch.load(bob.io.base.HDF5File(path),
shape=validation_shape, session=session)
scratch = SequenceNetwork()
scratch.load(session, os.path.join(directory, "model.ckp"))
[data, labels] = validation_data_shuffler.get_batch()
predictions = scratch(data, session=session)
accuracy = 100. * numpy.sum(numpy.argmax(predictions, 1) == labels) / predictions.shape[0]
......@@ -78,8 +73,6 @@ def test_cnn_trainer_scratch():
data_augmentation=data_augmentation)
validation_data = numpy.reshape(validation_data, (validation_data.shape[0], 28, 28, 1))
directory = "./temp/cnn"
# Create scratch network
scratch = scratch_network()