Initial commit

neural_net/activation_layers/activation_layer.py

@@ -0,0 +1,95 @@
+from abc import abstractmethod
+
+import numpy as np
+
+from neural_net.transform_layer import Layer
+
+class ActivationLayer(Layer):
+    def __init__(self, index, input_dim, output_dim, weights=None, biases=None):
+        super().__init__('ActivationLayer', index, input_dim, output_dim)
+        self.type = 'ActivationLayer'
+        self.subtype = ''
+
+        self.inputs = np.array([])
+        self.output = np.array([])
+        self.z = np.array([])
+        self.gradient_clip = 1.0
+
+        # Initialize weights and biases
+        if weights is not None:
+            self.weights = weights
+        else:
+            self.initialize_weights()
+        if biases is not None:
+            self.biases = biases
+        else:
+            self.initialize_biases()
+
+    def describe(self):
+        return f"{self.type} ({self.input_dim}x{self.output_dim} neurons, {self.subtype} activation)"
+
+    @abstractmethod
+    def initialize_weights(self):
+        pass
+
+    @abstractmethod
+    def initialize_biases(self):
+        pass
+
+    def forward(self, inputs: np.array):
+        self.inputs = inputs
+        self.z = np.dot(self.inputs, self.weights) + self.biases
+        self.output = self.activation(self.z)  # Calls the implemented class's activation function (ie. Sigmoid)
+        return self.output
+
+    def backward(self, dL_dout, learning_rate):
+        """
+        Backpropagate the error and update weights and biases.
+        :param dL_dout: Gradient of loss with respect to layer outputs
+        :param learning_rate: Learning rate for weight updates
+        :return: Gradient with respect to inputs for previous layer (dL/dinputs)
+        """
+        # Activation derivative dout/dz
+        # This tells you how much the output of the activation function changes with respect to the pre-activation value z.
+        # Sigmoid derivative formula: σ(z) * (1 - σ(z))
+        dout_dz = self.activation_derivative(self.output)
+
+        # Gradient of the loss with respect to weights (dL/dweights)
+        # This represents how much the loss changes when the weights change.
+        # Formula: dL/dweights = inputs × dL/dout × σ′(z)
+        dL_dweights = np.clip(np.dot(self.inputs.T, dL_dout * dout_dz), -self.gradient_clip, self.gradient_clip)
+
+        dL_dbias = np.sum(dL_dout * dout_dz, axis=0)
+
+        # Gradient of the loss with respect to inputs (dL/dinputs)
+        # This is the gradient of the loss with respect to the input of the neuron or layer, often needed if you want to backpropagate further.
+        # Formula: dL / dinputs = dL/dout × σ′(z) × weights
+        dL_dinputs = np.dot(dL_dout * dout_dz, self.weights.T)
+
+        # Clip gradients to prevent them from being too large
+        # np.clip(dL_dweights, -10.0, 10.0, out=dL_dweights)
+        # np.clip(dL_dbias, -10.0, 10.0, out=dL_dbias)
+
+        # Adjust weights and biases
+        self.weights -= learning_rate * dL_dweights
+        self.biases -= learning_rate * dL_dbias
+
+        return dL_dinputs, dL_dweights, dL_dbias, self.weights, self.biases
+
+    def reset(self):
+        self.initialize_weights()
+        self.initialize_biases()
+
+    @abstractmethod
+    def activation(self, raw_outputs: np.array):
+        """
+        Apply the activation function (Sigmoid, ReLU, etc.)
+        """
+        pass
+
+    @abstractmethod
+    def activation_derivative(self, outputs: np.array):
+        """
+        Compute the derivative of the activation function
+        """
+        pass

neural_net/activation_layers/relu_layer.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+from neural_net.activation_layers.activation_layer import ActivationLayer
+from neural_net.functions.activation import relu_activation, relu_derivative_activation
+
+
+class ReluLayer(ActivationLayer):
+    def __init__(self, index, input_dim, output_dim, weights=None, biases=None):
+        super().__init__(index, input_dim, output_dim, weights, biases)
+        self.subtype = 'RELU'
+
+    def initialize_weights(self):
+        # He initialization (input_dim x output_dim)
+        self.weights = np.random.randn(self.input_dim, self.output_dim) * np.sqrt(2.0 / self.input_dim)
+
+    def initialize_biases(self):
+        self.biases = np.zeros((1, self.output_dim))  # Biases initialized to zero
+
+    def activation(self, outputs: np.array):
+        return relu_activation(outputs)
+
+    def activation_derivative(self, outputs: np.array):
+        return relu_derivative_activation(outputs)

neural_net/activation_layers/sigmoid_layer.py

@@ -0,0 +1,24 @@
+import numpy as np
+
+from neural_net.activation_layers.activation_layer import ActivationLayer
+from neural_net.functions.activation import sigmoid_derivative_activation
+
+
+class SigmoidLayer(ActivationLayer):
+    def __init__(self, input_dim, output_dim, weights=None, biases=None):
+        super().__init__(input_dim, output_dim, weights, biases)
+        self.subtype = 'Sigmoid'
+
+    def initialize_weights(self):
+        # Xavier initialization for sigmoid activation
+        limit = np.sqrt(6 / (self.input_dim + self.output_dim))
+        self.weights = np.random.uniform(-limit, limit, (self.input_dim, self.output_dim))
+
+    def initialize_biases(self):
+        self.biases = np.zeros((1, self.output_dim))  # Biases initialized to zero
+
+    def activation(self, outputs: np.array):
+        return sigmoid_derivative_activation(outputs)
+
+    def activation_derivative(self, outputs: np.array):
+        return sigmoid_derivative_activation(outputs)