The code for generating a custom neural network and training it.
It has 3 Parts:
1)Activation Functions > Activation
2)Custom Layer generation > Layer
3)Gradient descent and backpropagation > Back_Pass
when I pass a single input, the network adapts well, but when I pass a batch of inputs the answers are totally wrong.
The error I doubt is in the cost evaluation of the back pass but I don't know what it is.
class Activations:
def Relu(self, input):
self.output = np.maximum(0, input)
self.deri = (self.output > 0).astype(int)
def Softmax(self, input):
input = input - np.max(input, axis = 1 , keepdims = True)
self.output = np.exp(input)/np.sum(np.exp(input),axis = 1, keepdims = True)
self.deri = self.output*(1- self.output)
class Layer(Activations):
def __init__(self, input_neurons, next_neurons, bias_req = 0):
self.weights = np.random.randn(input_neurons, next_neurons)
self.bias_req = bias_req
if bias_req == 1:
self.bais = np.random.randn(1,1)
else:
self.bais = [[0]]
def forward(self, inputs, activation):
self.inputs = np.array(inputs)
x = np.dot(self.inputs, self.weights) +self.bais
self.activation = activation
if activation == 'Relu':
self.Relu(x)
elif activation == "Softmax":
self.Softmax(x)
else:
self.output = x
self.deri = (self.output > self.output - 1).astype(int)
class Back_Pass:
def loss(self, expected, predicted):
self.cost = np.sum(0.5*(predicted - expected)**2, axis =0)/len(predicted)
self.error = np.sum((predicted - expected), axis = 0)/len(predicted)
def back(self, this_layer):
self.error = (this_layer.deri)* self.error
weights_buffer = this_layer.weights
if this_layer.bias_req == 1:
this_layer.bais -= l_rate*np.sum(self.error)
#Input maybe single or in batches
if len(self.error) == 1:
this_layer.weights -= l_rate*np.dot(this_layer.inputs.T, self.error)
else:
for i in range(len(self.error)):
this_layer.weights -= l_rate*np.dot(this_layer.inputs[i].T, self.error[i])
self.error = np.dot(self.error, weights_buffer.T)