Commit 5202a60f authored by Francois Marelli's avatar Francois Marelli

auto init

parent 7531241e
RANDOM_STD = 0 import numpy
EPSILON = 1e-6
INIT_MODULUS = 0.95
MIN_ANGLE = 0
MAX_ANGLE = numpy.pi / 2
def asig(x):
if not isinstance(x, np.ndarray):
x = np.array(x, ndmin=1)
x[x == 1] = 1 - EPSILON
x[x == 0] = EPSILON
return -numpy.log((1 / x) - 1)
def atanh(x):
if not isinstance(x, np.ndarray):
x = np.array(x, ndmin=1)
x[abs(x) == 1] *= (1 - EPSILON)
return numpy.arctanh(x)
from .log_loss import *
from .neural_filter import * from .neural_filter import *
from .neural_filter_1L import * from .neural_filter_1L import *
from .neural_filter_2R import *
from .neural_filter_2CC import * from .neural_filter_2CC import *
from .neural_filter_2CD import * from .neural_filter_2CD import *
from .log_loss import * from .neural_filter_2R import *
...@@ -30,7 +30,7 @@ from torch.nn import Parameter ...@@ -30,7 +30,7 @@ from torch.nn import Parameter
from torch.nn import functional as F from torch.nn import functional as F
import numpy as np import numpy as np
from . import RANDOM_STD from . import INIT_MODULUS, asig
class NeuralFilter(torch.nn.Module): class NeuralFilter(torch.nn.Module):
""" """
...@@ -48,15 +48,19 @@ class NeuralFilter(torch.nn.Module): ...@@ -48,15 +48,19 @@ class NeuralFilter(torch.nn.Module):
self.reset_parameters() self.reset_parameters()
def reset_parameters(self, init=None): def reset_parameters(self, init=None, min_modulus=0, max_modulus=1):
if init is None: if init is None:
self.bias_forget.data.uniform_(-RANDOM_STD, RANDOM_STD) parts = self.hidden_size * 2
else: ranges = np.arange(1, parts, 2)
if not isinstance(init, np.ndarray):
init = np.array(init, ndmin=1) init_modulus = ranges * (max_modulus - min_modulus) / parts + min_modulus
init = asig(init_modulus)
if not isinstance(init, np.ndarray):
init = np.array(init, ndmin=1)
ten_init = torch.from_numpy(init) ten_init = torch.from_numpy(init)
self.bias_forget.data.copy_(ten_init) self.bias_forget.data.copy_(ten_init)
def __repr__(self): def __repr__(self):
s = '{name}({hidden_size})' s = '{name}({hidden_size})'
......
...@@ -25,12 +25,12 @@ along with neural_filters. If not, see <http://www.gnu.org/licenses/>. ...@@ -25,12 +25,12 @@ along with neural_filters. If not, see <http://www.gnu.org/licenses/>.
""" """
import numpy as np
import torch import torch
from torch.nn import Parameter from torch.nn import Parameter
from torch.nn import functional as F from torch.nn import functional as F
import numpy as np
from . import RANDOM_STD from . import MIN_ANGLE, MAX_ANGLE, INIT_MODULUS, asig, atanh
class NeuralFilter2CC(torch.nn.Module): class NeuralFilter2CC(torch.nn.Module):
...@@ -50,26 +50,32 @@ class NeuralFilter2CC(torch.nn.Module): ...@@ -50,26 +50,32 @@ class NeuralFilter2CC(torch.nn.Module):
self.reset_parameters() self.reset_parameters()
def reset_parameters(self, init_modulus=None, init_theta=None): def reset_parameters(self, init_modulus=None, init_theta=None,
min_angle=MIN_ANGLE, max_angle=MAX_ANGLE, modulus=INIT_MODULUS):
if init_modulus is None: if init_modulus is None:
self.bias_modulus.data.uniform_(-RANDOM_STD, RANDOM_STD) init_modulus = asig(modulus)
else: if not isinstance(init_modulus, np.ndarray):
if not isinstance(init_modulus, np.ndarray): init_modulus = np.array(init_modulus, ndmin=1)
init_modulus = np.array(init_modulus, ndmin=1)
ten_init = torch.from_numpy(init_modulus) ten_init = torch.from_numpy(init_modulus)
self.bias_modulus.data.copy_(ten_init) self.bias_modulus.data.copy_(ten_init)
if init_theta is None: if init_theta is None:
self.bias_theta.data.uniform_(-RANDOM_STD, RANDOM_STD) parts = self.hidden_size * 2
ranges = np.arange(1, parts, 2)
else: init_angle = ranges * (max_angle - min_angle) / parts + min_angle
if not isinstance(init_theta, np.ndarray):
init_theta = np.array(init_theta, ndmin=1) cosangle = np.cos(init_angle)
init_theta = atanh(cosangle)
if not isinstance(init_theta, np.ndarray):
init_theta = np.array(init_theta, ndmin=1)
ten_init = torch.from_numpy(init_theta) ten_init = torch.from_numpy(init_theta)
self.bias_theta.data.copy_(ten_init) self.bias_theta.data.copy_(ten_init)
def __repr__(self): def __repr__(self):
s = '{name}({hidden_size})' s = '{name}({hidden_size})'
......
...@@ -25,10 +25,10 @@ along with neural_filters. If not, see <http://www.gnu.org/licenses/>. ...@@ -25,10 +25,10 @@ along with neural_filters. If not, see <http://www.gnu.org/licenses/>.
""" """
from . import NeuralFilter
import torch
import numpy as np import numpy as np
import torch
from . import NeuralFilter
class NeuralFilter2CD(torch.nn.Module): class NeuralFilter2CD(torch.nn.Module):
......
...@@ -30,8 +30,6 @@ from . import NeuralFilter ...@@ -30,8 +30,6 @@ from . import NeuralFilter
import torch import torch
import numpy as np import numpy as np
from . import RANDOM_STD
class NeuralFilter2R(torch.nn.Module): class NeuralFilter2R(torch.nn.Module):
""" """
...@@ -52,8 +50,9 @@ class NeuralFilter2R(torch.nn.Module): ...@@ -52,8 +50,9 @@ class NeuralFilter2R(torch.nn.Module):
def reset_parameters(self, init=None): def reset_parameters(self, init=None):
if init is None: if init is None:
self.first_cell.bias_forget.data.uniform_(-0.5 - RANDOM_STD, -0.5 + RANDOM_STD) self.first_cell.reset_parameters(min_modulus=0, max_modulus=0.5)
self.second_cell.bias_forget.data.uniform_(0.5 - RANDOM_STD, 0.5 + RANDOM_STD) self.second_cell.reset_parameters(min_modulus=0.5, max_modulus=1)
elif isinstance(init, tuple): elif isinstance(init, tuple):
self.first_cell.reset_parameters(init[0]) self.first_cell.reset_parameters(init[0])
self.second_cell.reset_parameters(init[1]) self.second_cell.reset_parameters(init[1])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment