I'm trying to learn backpropagation for the first time and I'm testing it with the xor problem, but I've been having the issue where the outputs always approach 0, regardless of input. Would anybody be able to tell me where I have problems in my algorithm? I have a bit of experience with neural nets but I've never attempted backpropagation before now so I'm a little lost.
backprop.pde
Matrix[] layers = new Matrix[3];
Matrix[] neuralnet = new Matrix[2];
public void setup() {
layers[0] = new Matrix(1,3);
layers[1] = new Matrix(1,2);
layers[2] = new Matrix(1,1);
neuralnet[0] = new Matrix(layers[0].cols, layers[1].cols);
neuralnet[1] = new Matrix(layers[1].cols, layers[2].cols);
for (Matrix i:neuralnet) {
i.randomize(-1,1);
}
}
public void draw() {
for (int i = 0; i < 100000; i++) {
int ip1 = round(random(1));
int ip2 = round(random(1));
boolean output = boolean(ip1) ^ boolean(ip2);
int out = int(output);
println(ip1, ip2, out);
layers[0].matrix[0][0] = ip1;
layers[0].matrix[0][1] = ip2;
layers[0].matrix[0][2] = 1;
layers = process(layers, neuralnet);
println(ip1, ip2, layers[2].matrix[0][0]);
Matrix expected = new Matrix(1,1);
expected.matrix[0][0] = out;
neuralnet = backpropagate(layers, neuralnet, expected);
//delay(1000);
}
for (Matrix i:neuralnet) {
println(i.toString());
}
exit();
}
public Matrix[] backpropagate(Matrix[] layers, Matrix[] weights, Matrix expected) {
float LR = .01;
Matrix[] errors = new Matrix[layers.length];
//calculate errors
for (int i = layers.length - 1; i >= 0; i--) {
if (i == layers.length - 1) {
errors[i] = layers[i].subtraction(expected);
} else {
errors[i] = layers[i+1].dotProduct(weights[i].Transpose());
}
errors[i] = errors[i].ElementMultiply(layers[i].activationPrime());
}
//adjust weights accordingly
Matrix[] newWeights = weights.clone();
for (int i = 0; i < newWeights.length; i++) {
newWeights[i] = newWeights[i].subtraction(layers[i].Transpose().dotProduct(errors[i+1]).scalarMultiply(LR));
}
return newWeights;
}
public Matrix[] process(Matrix[] layers, Matrix[] weights) {
for (int i = 1; i < layers.length; i++) {
layers[i] = layers[i-1].dotProduct(weights[i-1]);
}
return layers;
}
Matrix.pde
int rows, cols;
float[][] matrix;
Matrix(int i,int j) {
rows = i;
cols = j;
matrix = new float[i][j];
}
//----------------Matrix Functions----------------
void randomize(int min, int max) {
for(int tempRow = 0; tempRow < rows; tempRow++) {
for(int tempCol = 0; tempCol < cols; tempCol++) {
matrix[tempRow][tempCol] = random(min,max);
}
}
}
Matrix scalarMultiply(float scalar) {
Matrix result = new Matrix(this.rows, this.cols);
for(int tempRow = 0; tempRow < result.rows; tempRow++) {
for(int tempCol = 0; tempCol < result.cols; tempCol++) {
result.matrix[tempRow][tempCol] = this.matrix[tempRow][tempCol] * scalar;
}
}
return(result);
}
Matrix dotProduct(Matrix that) {
if (this.cols != that.rows) {
return(new Matrix(1,1));
}
Matrix result = new Matrix(this.rows,that.cols);
float tempSum;
for(int tempRow = 0; tempRow < result.rows; tempRow++) {
for(int tempCol = 0; tempCol < result.cols; tempCol++) {
tempSum = 0;
for(int inputTemp = 0; inputTemp < this.cols; inputTemp++) {
tempSum += this.matrix[tempRow][inputTemp] * that.matrix[inputTemp][tempCol];
}
result.matrix[tempRow][tempCol] = tempSum;
}
}
return(result);
}
Matrix addition(Matrix that) {
Matrix result = new Matrix(this.rows, this.cols);
for(int tempRow = 0; tempRow < result.rows; tempRow++) {
for(int tempCol = 0; tempCol < result.cols; tempCol++) {
result.matrix[tempRow][tempCol] = this.matrix[tempRow][tempCol] + that.matrix[tempRow][tempCol];
}
}
return(result);
}
Matrix subtraction(Matrix that) {
Matrix result = new Matrix(this.rows, this.cols);
for(int tempRow = 0; tempRow < result.rows; tempRow++) {
for(int tempCol = 0; tempCol < result.cols; tempCol++) {
result.matrix[tempRow][tempCol] = this.matrix[tempRow][tempCol] - that.matrix[tempRow][tempCol];
}
}
return(result);
}
Matrix ElementMultiply(Matrix that) {
Matrix result = new Matrix(this.rows, this.cols);
for(int tempRow = 0; tempRow < result.rows; tempRow++) {
for(int tempCol = 0; tempCol < result.cols; tempCol++) {
result.matrix[tempRow][tempCol] = this.matrix[tempRow][tempCol] * that.matrix[tempRow][tempCol];
}
}
return(result);
}
Matrix Transpose() {
Matrix result = new Matrix(this.cols, this.rows);
for(int tempRow = 0; tempRow < result.rows; tempRow++) {
for(int tempCol = 0; tempCol < result.cols; tempCol++) {
result.matrix[tempRow][tempCol] = this.matrix[tempCol][tempRow];
}
}
return result;
}
void activation() {
for(int tempRow = 0;tempRow < rows; tempRow++) {
for(int tempCol = 0; tempCol < cols; tempCol++) {
matrix[tempRow][tempCol] = sigmoid(matrix[tempRow][tempCol]);
}
}
}
Matrix activationPrime() {
Matrix result = new Matrix(this.rows, this.cols);
for(int tempRow = 0;tempRow < rows; tempRow++) {
for(int tempCol = 0; tempCol < cols; tempCol++) {
result.matrix[tempRow][tempCol] = sigmoidPrime(matrix[tempRow][tempCol]);
}
}
return result;
}
float sigmoid(float in) {
return (1 / (1 + exp(-1 * in)));
}
float sigmoidPrime(float in) {
return sigmoid(in) * (1 - sigmoid(in));
}
String toString() {
String out = "";
for(int tempRow = 0;tempRow < rows; tempRow++) {
for(int tempCol = 0; tempCol < cols; tempCol++) {
out += matrix[tempRow][tempCol] + " ";
}
out += "\n";
}
return out;
}
}