commit 8f776e61b2650486873516f903cb199e321e10f5
Author: Soikk <76824648+Soikk@users.noreply.github.com>
Date:   Mon Sep 5 14:31:44 2022 +0200

    Initial commit

diff --git a/image/image.c b/image/image.c
new file mode 100644
index 0000000..99d1341
--- /dev/null
+++ b/image/image.c
@@ -0,0 +1,53 @@
+#include "image.h"
+
+
+image *loadCSV(FILE *fp){
+	image *im = malloc(sizeof(image));
+	char line[MAXCHARS];
+	fgets(line, MAXCHARS, fp);
+	im->label = atoi(strtok(line, ","));
+	matrix *m = newMatrix(28, 28);
+	for(int i = 0; i < 28; ++i){
+		for(int j = 0; j < 28; ++j){
+			m->data[i][j] = strtod(strtok(NULL, ","), NULL)/256;
+		}
+	}
+	im->img = m;
+	return im;
+}
+
+void freeImage(image **im){
+	freeMatrix(&((*im)->img));
+	free(*im);
+	*im = NULL;
+}
+
+void printImage(image *im){
+	printf("%d, %d\n", im->img->rows, im->img->cols);
+	printf("%d\n", im->label);
+	for(int i = 0; i < im->img->rows; ++i){
+		for(int j = 0; j < im->img->cols; ++j){
+			printf("%c ", (im->img->data[i][j] > 0) ? '1' : 0 );
+		}
+		printf("\n");
+	}
+}
+/*int main(){
+
+	FILE *fp = fopen("../mnist_train.csv", "r");
+	char line[MAXCHARS];
+	fgets(line, MAXCHARS, fp);
+	image *im = loadMnist_train(fp);
+	
+	printf("%d, %d\n", im->img->rows, im->img->cols);
+	printf("%d\n", im->label);
+	for(int i = 0; i < im->img->rows; ++i){
+		if(i%28 == 0)
+			printf("\n");
+		for(int j = 0; j < im->img->cols; ++j){
+			printf("%c ", (im->img->data[i][j] > 0) ? '1' : 0 );
+		}
+	}
+
+	return 0;
+}*/
diff --git a/image/image.h b/image/image.h
new file mode 100644
index 0000000..a5446a8
--- /dev/null
+++ b/image/image.h
@@ -0,0 +1,25 @@
+#pragma once
+#ifndef IMAGE_H
+#define IMAGE_H
+
+#define __MINGW_FEATURES__ 1
+
+#include <string.h>
+#include "../../matrix/matrix.h"
+
+#define MNIST_SIZE 784
+#define MAXCHARS 10000
+
+
+typedef struct image{
+	int label;
+	matrix *img;
+} image;
+
+image *loadCSV(FILE *fp);
+
+void freeImage(image **im);
+
+void printImage(image *im);
+
+#endif
diff --git a/image/main.exe b/image/main.exe
new file mode 100644
index 0000000..4d6969d
Binary files /dev/null and b/image/main.exe differ
diff --git a/neural.c b/neural.c
new file mode 100644
index 0000000..0942569
--- /dev/null
+++ b/neural.c
@@ -0,0 +1,410 @@
+#include "neural.h"
+
+
+void printMatrix(matrix *m){
+	printf("%dx%d\n", m->rows, m->cols);
+	for(int i = 0; i < m->rows; ++i){
+		for(int j = 0; j < m->cols; ++j){
+			printf("%.3Lf ", m->data[i][j]);
+		}
+		printf("\n");
+	}
+}
+
+void printLayer(layer *l){
+	printf("function: %d, inputs: %d, nneurons: %d\n", l->function, l->inputs, l->nneurons);
+	printf("Weights\n");
+	printMatrix(l->weights);
+	printf("Bias\n");
+	printMatrix(l->bias);
+	printf("Neurons\n");
+	printMatrix(l->neurons);
+}
+
+void printNet(net *n){
+	printf("learningrate: %.3Lf, inputs: %d, outputs, %d, nlayers: %d\n", n->learningrate, n->inputs, n->outputs, n->nlayers);
+	printf("input:\n");
+	printMatrix(n->input);
+	for(int i = 0; i < n->nlayers; ++i){
+		printf("Layer %d:\n", i);
+		printLayer(n->layers[i]);
+	}
+}
+
+static long double linear(long double n){ return n; }
+
+static long double derivedLinear(long double n){ return 0.0; }
+
+static long double ReLu(long double n){ return fmaxl(0.0, n); }
+
+static long double derivedReLu(long double n){ return n > 0; }
+
+static long double sigmoid(long double n){ return 1/(1 + expl(-n)); }
+
+static long double derivedSigmoid(long double n){ return n*(1 - n); }
+
+static long double derivedTanhl(long double n){ return 1 - tanhl(n)*tanhl(n); }
+
+static long double he(long double inputs){
+	long long int scale = 10000000000;
+	int r = rand()%(int)(sqrtl(2.0/inputs)*scale);
+	return (long double)(r/scale);
+}
+
+static long double xavier(long double inputs){
+}
+
+
+static long double (*functions[])(long double) = {
+	linear, ReLu, sigmoid, tanhl,
+};
+
+static long double (*derivedFunctions[])(long double) = {
+	derivedLinear,derivedReLu, derivedSigmoid, derivedTanhl,
+};
+
+static long double placeholder(long double n){
+	long double high = 1/sqrtl(n), low = (-1)/sqrtl(n);
+	long double difference = high - low; // The difference between the two
+	int scale = 10000;
+	int scaled_difference = (int)(difference * scale);
+	return low + (1.0 * (rand() % scaled_difference) / scale);
+}
+
+// Rework
+void initializeLayer(layer *l){
+	srand(time(NULL));
+	// TODO implement different initialization functions (he, xavier)
+	for(int i = 0; i < l->weights->rows; ++i){
+		for(int j = 0; j < l->weights->cols; ++j){
+			l->weights->data[i][j] = placeholder(l->nneurons);
+		}
+	}
+}
+
+static layer *newLayer(FUNCTIONS function, int inputs, int nneurons){
+	layer *l = malloc(sizeof(layer));
+	l->function = function,
+	l->inputs = inputs;
+	l->nneurons = nneurons;
+	l->weights = newMatrix(inputs, nneurons);
+	initializeLayer(l);
+	l->bias = newMatrix(1, nneurons);
+	fillMatrix(l->bias, 0);
+	l->neurons = newMatrix(1, nneurons);
+	fillMatrix(l->neurons, 0);
+	return l;
+}
+
+static void freeLayer(layer **l){
+	freeMatrix(&(*l)->weights);
+	freeMatrix(&(*l)->bias);
+	freeMatrix(&(*l)->neurons);
+	free(*l);
+	l = NULL;
+}
+
+static void saveLayer(layer *l, FILE *fp){
+	char header = 'L';
+	fwrite(&header, sizeof(char), 1, fp);
+	fwrite(&l->function, sizeof(int), 1, fp);
+	fwrite(&l->inputs, sizeof(int), 1, fp);
+	fwrite(&l->nneurons, sizeof(int), 1, fp);
+	saveMatrix(l->weights, fp);
+	saveMatrix(l->bias, fp);
+	saveMatrix(l->neurons, fp);
+	char end = 'E';
+	fwrite(&end, sizeof(char), 1, fp);
+}
+
+static layer *loadLayer(FILE *fp){
+	char header;
+	fread(&header, sizeof(char), 1, fp);
+	if(header != 'L'){
+		fprintf(stderr, "Header is '%c' not 'L'\n", header);
+		exit(EXIT_FAILURE);
+	}
+	FUNCTIONS function;
+	int inputs, nneurons;
+	fread(&function, sizeof(int), 1, fp);
+	fread(&inputs, sizeof(int), 1, fp);
+	fread(&nneurons, sizeof(int), 1, fp);
+	layer *l = malloc(sizeof(layer));
+	l->function = function;
+	l->inputs = inputs;
+	l->nneurons = nneurons;
+	l->weights = loadMatrix(fp);
+	l->bias = loadMatrix(fp);
+	l->neurons = loadMatrix(fp);
+	char end;
+	fread(&end, sizeof(char), 1, fp);
+	if(end != 'E'){
+		fprintf(stderr, "End is '%c' not 'E'\n", end);
+		exit(EXIT_FAILURE);
+	}
+	return l;
+}
+
+net *newNet(FUNCTIONS function, long double learningrate, int inputs, int outputs, int nlayers, ...){
+	// TODO check if outputs == last layer
+	net *n = malloc(sizeof(net));
+	n->learningrate = learningrate;
+	n->inputs = inputs;
+	n->outputs = outputs;
+	n->nlayers = nlayers;
+	n->input = newMatrix(1, inputs);
+	fillMatrix(n->input, 1);
+	n->layers = malloc(nlayers*sizeof(layer*));
+	va_list layers;
+	va_start(layers, nlayers);
+	for(int i = 0; i < nlayers; ++i){
+		int size = va_arg(layers, int);
+		n->layers[i] = newLayer(function, inputs, size);
+		inputs = size;
+	}
+	va_end(layers);
+	return n;
+}
+
+void freeNet(net **n){
+	freeMatrix(&(*n)->input);
+	for(int i = 0; i < (*n)->nlayers; ++i){
+		freeLayer(&(*n)->layers[i]);
+	}
+	free((*n)->layers);
+	(*n)->layers = NULL;
+	free(*n);
+	n = NULL;
+}
+
+void saveNet(net *n, FILE *fp){
+	char header = 'N';
+	fwrite(&header, sizeof(char), 1, fp);
+	fwrite(&n->learningrate, sizeof(long double), 1, fp);
+	fwrite(&n->inputs, sizeof(int), 1, fp);
+	fwrite(&n->outputs, sizeof(int), 1, fp);
+	fwrite(&n->nlayers, sizeof(int), 1, fp);
+	saveMatrix(n->input, fp);
+	for(int i = 0; i < n->nlayers; ++i){
+		saveLayer(n->layers[i], fp);
+	}
+	char end = 'E';
+	fwrite(&end, sizeof(char), 1, fp);
+}
+
+net *loadNet(FILE *fp){
+	char header;
+	fread(&header, sizeof(char), 1, fp);
+	if(header != 'N'){
+		fprintf(stderr, "Header is '%c' not 'N'\n", header);
+		exit(EXIT_FAILURE);
+	}
+	long double learningrate;
+	int inputs, outputs, nlayers;
+	fread(&learningrate, sizeof(long double), 1, fp);
+	fread(&inputs, sizeof(int), 1, fp);
+	fread(&outputs, sizeof(int), 1, fp);
+	fread(&nlayers, sizeof(int), 1, fp);
+	net *n = malloc(sizeof(net));
+	n->learningrate = learningrate;
+	n->inputs = inputs;
+	n->outputs = outputs;
+	n->nlayers = nlayers;
+	n->input = loadMatrix(fp);
+	n->layers = malloc(nlayers*sizeof(layer*));
+	for(int i = 0; i < nlayers; ++i){
+		n->layers[i] = loadLayer(fp);
+	}
+	char end;
+	fread(&end, sizeof(char), 1, fp);
+	if(end != 'E'){
+		fprintf(stderr, "End is '%c' not 'E'\n", end);
+		exit(EXIT_FAILURE);
+	}
+	return n;
+}
+
+static void applyFunction(func function, matrix *m){
+	for(int i = 0; i < m->rows; ++i){
+		for(int j = 0; j < m->cols; ++j){
+			m->data[i][j] = function(m->data[i][j]);
+		}
+	}
+}
+
+static void propagateLayer(layer *l, matrix *inputs){
+	matrix *m = multiplyMatrices(inputs, l->weights);
+	matrix *a = addMatrices(m, l->bias);
+	freeMatrix(&m);
+	copyMatrix(l->neurons, a);
+	freeMatrix(&a);
+	applyFunction(functions[l->function], l->neurons);
+}
+
+matrix *propagate(net *n, matrix *input){
+	n->input = input;
+	for(int i = 0; i < n->nlayers; ++i){
+		propagateLayer(n->layers[i], input);
+		input = n->layers[i]->neurons;
+	}
+	return n->layers[n->nlayers-1]->neurons;
+}
+
+void backPropagate(net *n, matrix *expected){
+	matrix **errors = malloc(n->nlayers*sizeof(matrix*));
+	matrix *corrected = subtractMatrices(expected, n->layers[n->nlayers-1]->neurons);
+
+	for(int i = n->nlayers-1; i >= 0; --i){
+		matrix *derived = cloneMatrix(n->layers[i]->neurons);
+		applyFunction(derivedFunctions[n->layers[i]->function], derived);
+		errors[i] = HadamardProduct(corrected, derived);
+		freeMatrix(&corrected);
+		freeMatrix(&derived);
+		
+		matrix *transposedWeights = transpose(n->layers[i]->weights);
+		corrected = multiplyMatrices(errors[i], transposedWeights);
+		freeMatrix(&transposedWeights);
+	}
+	
+	matrix *lastOutput = n->input;
+	for(int i = 0; i < n->nlayers; ++i){
+		matrix *transposedOutput = transpose(lastOutput);
+		multiplyMatrix(transposedOutput, n->learningrate);
+		matrix *weightChangeMatrix = multiplyMatrices(transposedOutput, errors[i]);
+		freeMatrix(&transposedOutput);
+		
+		matrix *t = addMatrices(n->layers[i]->weights, weightChangeMatrix);
+		freeMatrix(&weightChangeMatrix);
+		copyMatrix(n->layers[i]->weights, t);
+		freeMatrix(&t);
+		
+		multiplyMatrix(errors[i], n->learningrate);
+		t = addMatrices(n->layers[i]->bias, errors[i]);
+		copyMatrix(n->layers[i]->bias, t);
+		freeMatrix(&t);
+		
+		lastOutput = n->layers[i]->neurons;
+	}
+	lastOutput = NULL;
+
+	for(int i = 0; i < n->nlayers; ++i){
+		freeMatrix(&errors[i]);
+	}
+	free(errors);
+	errors = NULL;
+	freeMatrix(&corrected);
+}
+
+void feedData(matrix *m, long double array[m->rows][m->cols]){
+	for(int i = 0; i < m->rows; ++i){
+		for(int j = 0; j < m->cols; ++j){
+			m->data[i][j] = array[i][j];
+		}
+	}
+}
+
+matrix *imageToInput(image *im){
+	matrix *r = newMatrix(1, im->img->rows*im->img->cols);
+	int l = 0;
+	for(int i = 0; i < im->img->rows; ++i){
+		for(int j = 0; j < im->img->cols; ++j){
+			r->data[0][l++] = im->img->data[i][j];
+		}
+	}
+	return r;
+}
+
+static int maxOutputs(matrix *out){
+	long double max = 0;
+	int mi = -1;
+	for(int i = 0; i < 10; ++i){
+		if(out->data[0][i] > max){
+			max = out->data[0][i];
+			mi = i;
+		}
+	}
+	return mi;
+}
+
+int main(){
+
+	matrix **expectedOutputs = malloc(10*sizeof(matrix*));
+	for(int i = 0; i < 10; ++i){
+		expectedOutputs[i] = newMatrix(1, 10);
+		fillMatrix(expectedOutputs[i], 0);
+		expectedOutputs[i]->data[0][i] = 1;
+	}
+
+	// Training
+	net *n = newNet(SIGMOID, 1, 784, 10, 2, 300, 10);
+	
+	int training = 0;
+	FILE *trainFP = fopen("mnist_train.csv", "r");
+	char line[MAXCHARS];
+	fgets(line, MAXCHARS, trainFP);
+	/*while(!feof(trainFP)){
+		image *im = loadCSV(trainFP);
+		propagate(n, imageToInput(im));
+		backPropagate(n, expectedOutputs[im->label]);
+		
+		if(training%100 == 0){
+			printf("\nTrained %d\n", training+1);
+			printMatrix(n->layers[n->nlayers-1]->neurons);
+			printf("\n");
+		}
+		
+		freeImage(&im);
+		++training;
+	}
+	fclose(trainFP);
+	printMatrix(n->input);
+	
+	FILE *fp = fopen("nn", "wb");
+	saveNet(n, fp);
+	fclose(fp);*/
+	freeNet(&n);
+	
+	FILE *fp = fopen("nn", "rb");
+	n = loadNet(fp);
+	fclose(fp);
+
+	// Testing
+
+	int testing = 0;
+	FILE *testFP = fopen("mnist_test.csv", "r");
+	//char line[MAXCHARS];
+	fgets(line, MAXCHARS, testFP);
+	
+	int count = 0;
+	while(!feof(testFP)){
+		image *im = loadCSV(testFP);
+		matrix *in = imageToInput(im);
+		matrix *expectedOutput = expectedOutputs[im->label];
+		matrix *output = propagate(n, in);
+		if(testing%300 == 0){
+			printf("Tested %d so far\n", testing+1);
+		}
+		printImage(im);
+		printf("Expected:\n");
+		printMatrix(expectedOutput);
+		printf("Got:\n");
+		printMatrix(output);
+		printf("Max output: %d\n", maxOutputs(output));
+		if(maxOutputs(output) == im->label){
+			count++;
+		}
+		freeMatrix(&in);
+		output = NULL;
+		freeImage(&im);
+		++testing;
+		if(testing == 10)
+			break;
+	}
+	fclose(testFP);
+
+	long double success = ((long double)count/testing)*100;
+	printf("success rate: %Lf (%d/%d)\n", success, count, testing);
+	
+
+	return 0;
+}
diff --git a/neural.h b/neural.h
new file mode 100644
index 0000000..501055e
--- /dev/null
+++ b/neural.h
@@ -0,0 +1,55 @@
+#pragma once
+#ifndef NEURAL_H
+#define NEURAL_H
+
+#define __MINGW_FEATURES__ 1
+
+#include <math.h>
+#include <time.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include "../matrix/matrix.h"
+#include "image/image.h"
+
+
+typedef enum{
+	LINEAR, RELU, SIGMOID, TANH,
+	//HE, XAVIER,
+} FUNCTIONS;
+
+typedef struct layer{
+	FUNCTIONS function;
+	int inputs;
+	int nneurons;
+	matrix *weights;
+	matrix *bias;
+	matrix *neurons;
+} layer;
+
+typedef struct net{
+	long double learningrate;
+	int inputs;
+	int outputs;
+	int nlayers;
+	matrix *input;
+	layer **layers;
+} net;
+
+typedef long double (*func)(long double);
+
+
+net *newNet(FUNCTIONS function, long double learningrate, int inputs, int outputs, int nlayers, ...);
+
+void saveNet(net *n, FILE *fp);
+
+net *loadNet(FILE *fp);
+
+matrix *propagate(net *n, matrix *input);
+
+void backPropagate(net *n, matrix *expected);
+
+void feedData(matrix *m, long double array[m->rows][m->cols]);
+
+matrix *imageToInput(image *im);
+
+#endif
diff --git a/nn b/nn
new file mode 100644
index 0000000..ef77373
Binary files /dev/null and b/nn differ