I wrote my neural network in c++ using MNIST dataset with numbers. but something goes wrong, and the neural network simply does not learn, giving random values at the output. at the input, I submit an array of 784 neurons (28X28 pixels in MNIST pictures). then I have 2 intermediate layers of 128 and 16 elements. as an activation function, I use RELU, only slightly modified (I also tried with sigmoid, nothing came out). help me find the error
#include <iostream>
#include <string>
#include <fstream>
#include <vector>
#include <math.h>
#include <cstdlib>
#define _AMOUNT_OF_LAYERS 4
#define _PATH_OF_IMAGES "train-images.idx3-ubyte"
#define _PATH_OF_LABELS "train-labels.idx1-ubyte"
#define _BATCH = 100
#define _SIZES_OF_LAYERS {784,128,16,10};
using namespace std;
struct Layer {
vector<long double>neurons;
vector<long double>biases;
vector<vector<long double>>weights;
};
int fromBigToLittleEndian(int i)
{
unsigned char ch1, ch2, ch3, ch4;
ch1 = i & 255;
ch2 = (i >> 8) & 255;
ch3 = (i >> 16) & 255;
ch4 = (i >> 24) & 255;
return ((int)ch1 << 24) + ((int)ch2 << 16) + ((int)ch3 << 8) + ch4;
}
void readImages(vector<vector<long double> >& Images) {
ifstream file;
file.open(_PATH_OF_IMAGES, ios::binary);
if (file.is_open())
{
int magicNumber = 0;
int numOfImages = 0;
int Rows = 0;
int Cols = 0;
file.read((char*)&magicNumber, sizeof(magicNumber));
magicNumber = fromBigToLittleEndian(magicNumber);
file.read((char*)&numOfImages, sizeof(numOfImages));
numOfImages = fromBigToLittleEndian(numOfImages);
file.read((char*)&Rows, sizeof(Rows));
Rows = fromBigToLittleEndian(Rows);
file.read((char*)&Cols, sizeof(Cols));
Cols = fromBigToLittleEndian(Cols);
for (int i = 0; i < numOfImages; i++)
{
Images.push_back({});
for (int r = 0; r < (Rows * Cols); r++)
{
unsigned char pixel = 0;
file.read((char*)&pixel, sizeof(pixel));
Images[i].push_back(pixel/255.0);
}
}
}
}
void readLabels(vector<long double>& Labels) {
ifstream file;
file.open(_PATH_OF_LABELS, ios::binary);
if (file.is_open())
{
int magicNumber = 0;
int numOfLabels = 0;
file.read((char*)&magicNumber, sizeof(magicNumber));
magicNumber = fromBigToLittleEndian(magicNumber);
file.read((char*)&numOfLabels, sizeof(numOfLabels));
numOfLabels = fromBigToLittleEndian(numOfLabels);
for (int i = 0; i < numOfLabels; i++)
{
unsigned char label = 0;
file.read((char*)&label, sizeof(label));;
Labels.push_back(label);
}
cout << magicNumber << endl << numOfLabels << endl;
}
}
long double sigmoid(long double x) {
return (1.0 / (1 + exp(-x)));
}
long double modRelu(long double x) {
if (x < 0) return 0.01 * x;
if (x >= 0 and x <= 1) return x;
if (x > 1) return 1 + 0.01 * (x - 1);
}
long double derivative(long double x) {
if (x < 0) return 0.01;
if (x >= 0 and x <= 1) return 1;
if (x > 1) return 0.01;
}
void DoLayers(vector<Layer>& layers, vector<int> sizes) {
for (int i = 0; i < sizes.size(); i++) {
int nextsize = 0;
if (i < sizes.size() - 1) nextsize = sizes[i + 1];
layers[i].neurons.resize(sizes[i]);
layers[i].biases.resize(sizes[i]);
layers[i].weights.resize(sizes[i]);
for (int q = 0; q < sizes[i]; q++) {
layers[i].weights[q].resize(nextsize);
}
for (int j = 0; j < sizes[i]; j++) {
if(rand()%2==0)
layers[i].biases[j] = (rand()%100)/100.0;
else
layers[i].biases[j] = -(rand() % 100)/100.0;
for (int k = 0; k < nextsize; k++) {
if (rand() % 2 == 0)
layers[i].weights[j][k] = (rand() % 100)/100.0;
else layers[i].weights[j][k] = -(rand() % 100) / 100.0;
}
}
}
}
void forwardFeeding(vector<long double>input, vector<Layer>& layers) {
for (int j = 0; j < input.size(); j++) {
layers[0].neurons[j] = (input[j]);
}
for (int i = 1; i < layers.size(); i++) {
Layer l = layers[i - 1];
Layer l1 = layers[i];
for (int j = 0; j < l1.neurons.size(); j++) {
l1.neurons[j] = 0;
for (int k = 0; k < l.neurons.size(); k++) {
layers[i].neurons[j] += layers[i - 1].neurons[k] * layers[i - 1].weights[k][j];
}
layers[i].neurons[j] += layers[i].biases[j];
layers[i].neurons[j] = modRelu(layers[i].neurons[j]);
}
}
}
void backPropagation(int predict, vector<Layer>& layers, double learningRate) {
vector<long double>errors;
errors.resize(10);
for (int i = 0; i < 10; i++) {
if (i == predict) errors[i] = (1.0 - layers[3].neurons[i])*derivative(1.0 - layers[3].neurons[i]);
else errors[i] = (0.0 - layers[3].neurons[i])*derivative(0.0 - layers[3].neurons[i]);
}
for (int k = 2; k >= 0; k--) {
vector<long double>errsNext;
vector<long double>gradients;
errsNext.resize(layers[k].neurons.size());
gradients.resize(layers[k + 1].neurons.size());
for (int i = 0; i < layers[k + 1].neurons.size(); i++) {
gradients[i] = errors[i] * derivative(layers[k + 1].neurons[i]);
gradients[i] *= learningRate;
}
vector<vector<long double> > delta;
delta.resize(layers[k + 1].neurons.size());
for (int i = 0; i < layers[k + 1].neurons.size(); i++) {
delta[i].resize(layers[k].neurons.size());
}
for (int i = 0; i < layers[k + 1].neurons.size(); i++) {
for (int j = 0; j < layers[k].neurons.size(); j++) {
delta[i][j] = gradients[i] * layers[k].neurons[j];
}
}
for (int i = 0; i < layers[k].neurons.size(); i++) {
errsNext[i] = 0;
for (int j = 0; j < layers[k + 1].neurons.size(); j++) {
errsNext[i] += layers[k].weights[i][j] * errors[j];
}
}
errors.resize(0);
errors.resize(layers[k].neurons.size());
for (int i = 0; i < layers[k].neurons.size(); i++) {
errors[i] = errsNext[i];
}
vector<vector<long double> > newWeights;
newWeights.resize(layers[k].neurons.size());
for (int i = 0; i < newWeights.size(); i++) {
newWeights[i].resize(layers[k+1].neurons.size());
}
for (int i = 0; i < layers[k+1].neurons.size(); i++) {
for (int j = 0; j < layers[k].neurons.size(); j++) {
newWeights[j][i] = (layers[k].weights[j][i] + delta[i][j]);
}
}
for (int i = 0; i < layers[k].weights.size(); i++) {
for (int j = 0; j < layers[k].weights[i].size(); j++) {
layers[k].weights[i][j] = newWeights[i][j];
}
}
for (int i = 0; i < layers[k + 1].biases.size(); i++) {
layers[k + 1].biases[i] += gradients[i];
}
}
}
int predict(vector<Layer>& layers) {
long double maxWeight = 0;
int digit = 0;
for (int i = 0; i < 10; i++) {
if (layers[layers.size() - 1].neurons[i] > maxWeight) {
maxWeight = layers[layers.size() - 1].neurons[i];
digit = i;
}
}
return digit;
}
int main() {
vector<vector<long double> >arrayOfImages;
vector<long double>arrayOfLabels;
vector<int> sizes = _SIZES_OF_LAYERS;
vector<Layer> layers;
readImages(arrayOfImages);
readLabels(arrayOfLabels);
layers.resize(_AMOUNT_OF_LAYERS);
int batch = 100;
int count = 0;
int epochs = 60;
int indexOfImg = 0;
DoLayers(layers, sizes);
for (int i = 0; i < epochs; i++) {
int count = 0;
long double errSum = 0;
for (int j = 0; j < batch; j++) {
int rightDigit = arrayOfLabels[indexOfImg];
forwardFeeding(arrayOfImages[indexOfImg], layers);
int predictedDigit = predict(layers);
if (predictedDigit == rightDigit) count++;
for (int i = 0; i < 10; i++) {
if (i == rightDigit) errSum += pow(1.0 - layers[3].neurons[i], 2);
else errSum += pow(0.0 - layers[3].neurons[i], 2);
}
backPropagation(predictedDigit, layers, 0.001);
indexOfImg++;
}
cout << "epoch: " << i << " right: " << count << "/100 " << "error: " << errSum << endl;
}
}