Initial commit

neural_net/functions/activation.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+def relu_activation(outputs):
+    return np.maximum(0, outputs)
+
+def relu_derivative_activation(outputs):
+    return np.where(outputs > 0, 1, 0)
+
+def sigmoid_activation(outputs):
+    return 1 / (1 + np.exp(-outputs))
+
+def sigmoid_derivative_activation(outputs):
+    return outputs * (1 - outputs)

neural_net/functions/loss.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+def cross_entropy_loss(outputs, targets, clip=True):
+    """
+    outputs: [
+        [ 0.32, 0.12, 0.04 ],
+        [ 0.62, 0.02, 0.14 ]
+    ]
+    targets: [ 2, 1 ]
+    :param outputs: np.array: Vector of all the predicted probabilities vectors
+    :param targets: np.array: Vector of one-hot vectors representing the actual values
+    :param clip: boolean, whether to clip the output probabilities
+    :return:
+    """
+    if clip:
+        # Clipping the predictions for numerical stability
+        outputs = np.clip(outputs, 1e-12, 1 - 1e-12)
+    # Calculate cross-entropy loss and average over batch size
+    m = targets.shape[0]
+    log_likelihood = -np.log(outputs[range(m), targets])
+    return np.sum(log_likelihood) / m  # Average loss
+
+def cross_entropy_derivative_loss(outputs, targets):
+    # One-hot encode the labels
+    y_true = np.eye(outputs.shape[1])[targets]
+    # Derivative of cross-entropy with respect to softmax inputs
+    return outputs - y_true