Neural.py

# A Neural Network from scratch using only math and numpy.

import numpy as np
import os
import pickle

class NeuralNetwork:
    def __init__(self, layer_sizes, activation="relu", learning_rate=0.01):
        self.layer_sizes = layer_sizes
        self.num_layers = len(layer_sizes)
        self.learning_rate = learning_rate
        self.activation = activation
         
        # Initialize weights and biases
        self.weights = []
        self.biases = []
         
        # He initialization for weights (works well with ReLu)
        for i in range(self.num_layers - 1):
            w = np.random.randn(layer_sizes[i], layer_sizes[i + 1]) * np.sqrt(2.0 / layer_sizes[i])
            b = np.zeros((1, layer_sizes[i + 1]))
            self.weights.append(w)
            self.biases.append(b)
           
    def _activate(self, z):
        # Apply activation function
         
        if self.activation == "relu":
            return np.maximum(0, z)
        elif self.activation == "sigmoid":
            return 1 / (1 + np.exp(-np.clip(z, -500, 500)))
        elif self.activation == "tanh":
            return np.tanh(z)
        else:
            raise ValueError(f"Uknown Activation: {self.activation}")
         
    def _activate_derivative(self, z):
        # Compute derivative of activation function
         
        if self.activation == "relu":
            return (z > 0).astype(float)
        elif self.activation == "sigmoid":
            sig = 1 / (1 + np.exp(-np.clip(z, -500, 500)))
            return sig * (1 - sig)
        elif self.activation == "tanh":
            return 1 - np.tanh(z) ** 2
        else:
            raise ValueError(f"Unknown Activation: {self.activation}")
        
    def _softmax(self, z):
        # Softmax activation for output layer
        
        exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))
        return exp_z / np.sum(exp_z, axis=1, keepdims=True)
        
    def forward(self, X):
        # Forward propagation through the network
        # Parameters: X : numpy array of shape (n_samples, n_features) - Input data
        # Returns: activations : list of numpy arrays - Activations at each layer (including input)
        # ^ Pre-Activation values at each number
     
        activations = [X]
        z_values = []
     
        # Forward through hidden layers
        for i in range(self.num_layers - 2):
            z = np.dot(activations[-1], self.weights[i]) + self.biases[i]
            z_values.append(z)
            a = self._activate(z)
            activations.append(a)
        
        # Output layer with softmax
        z = np.dot(activations[-1], self.weights[-1]) + self.biases[-1]
        z_values.append(z)
        a = self._softmax(z)
        activations.append(a)
     
        return activations, z_values
     
    def backward(self, X, y, activations, z_values):
        m = X.shape[0] # Number of samples
        weight_gradients = []
        bias_gradients = []
     
        # Output layer gradient (softmax + cross-entropy)
        delta = activations[-1] - y
     
        # Backpropagate through layers
        for i in range(self.num_layers - 2, -1, -1):
            # Compute gradients
            dw = np.dot(activations[i].T, delta) / m
            db = np.sum(delta, axis=0, keepdims=True) / m
         
            weight_gradients.insert(0, dw)
            bias_gradients.insert(0, db)
         
            # Propagate error to the previous layer (if not input layer)
            if i > 0:
                delta = np.dot(delta, self.weights[i].T) * self._activate_derivative(z_values[i - 1])
         
        return weight_gradients, bias_gradients
     
    def train(self, X, y, epochs=500, batch_size=32, verbose=True):
     
        # Convert lables to one-hot encoding if needed
        if len(y.shape) == 1:
            y_one_hot = np.zeros((y.shape[0], self.layer_sizes[-1]))
            y_one_hot[np.arange(y.shape[0]), y] = 1
        else:
            y_one_hot = y
            
        n_samples = X.shape[0]
         
        for epoch in range(epochs):
            # Shuffle Data
         
            indices = np.random.permutation(n_samples)
            X_shuffled = X[indices]
            y_shuffled = y_one_hot[indices]
         
            # Mini batch training
            for i in range(0, n_samples, batch_size):
                X_batch = X_shuffled[i:i + batch_size]
                y_batch = y_shuffled[i:i + batch_size]
             
                # Forward Pass
                activations, z_values = self.forward(X_batch)
             
                # Backward Pass
                weight_gradients, bias_gradients = self.backward(X_batch, y_batch, activations, z_values)
             
                # Update Weights and Biases
                for j in range(len(self.weights)):
                    self.weights[j] -= self.learning_rate * weight_gradients[j]
                    self.biases[j] -= self.learning_rate * bias_gradients[j]
                  
                 
            # Print Progress
            if verbose and (epoch + 1) % 10 == 0:
                activations, _ = self.forward(X)
                loss = self._cross_entropy_loss(y_one_hot, activations[-1])
                accuracy = self.accuracy(X, y if len(y.shape) == 1 else np.argmax(y, axis=1))
                print(f"Epoch {epoch + 1}/{epochs} - Loss: {loss:.4f} - Accuracy: {accuracy:.4f}")
                    
    def _cross_entropy_loss(self, y_true, y_pred):
        # Compute cross-entropy loss
        m = y_true.shape[0]
        log_likelihood = -np.log(y_pred[range(m), np.argmax(y_true, axis=1)] + 1e-8)
        return np.sum(log_likelihood) / m
      
    def predict(self, X):
        # Make predictions on new data
        activations, _ = self.forward(X)
        return np.argmax(activations[-1], axis=1)
     
    def predict_proba(self, X):
        # Predict class probabilities
        activations, _ = self.forward(X)
        return activations[-1]
     
    def accuracy(self, X, y):
        # Compute accuracy on given data
     
        predictions = self.predict(X)
        return np.mean(predictions == y)
     
    def save(self, filename):
        import pickle
        with open(filename, "wb") as f:
            pickle.dump(self, f)
     
    @staticmethod
    def load(filename):
        import pickle
        with open(filename, "rb") as f:
            return pickle.load(f)
         
# Example Usage 
if __name__ == "__main__":
    
    # Generate datasets
    from sklearn.datasets import load_digits, load_wine, load_iris
    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import StandardScaler

    # Create Dataset
    # digits = load_digits()
    iris = load_iris()
    X, y = iris.data, iris.target
    # X, y = digits.data, digits.target
    # Split and Normalize

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    # Create and Train Neural Network
    print("Training Neural Network...")
    if os.path.exists("model.pkl"):
        print("Loading Existing Model...")
        nn = NeuralNetwork.load("model.pkl")
    else:
        print("Creating New Model...")
        nn = NeuralNetwork(layer_sizes=[4, 8, 6, 3], activation="relu", learning_rate=0.01)
     
    nn.train(X_train, y_train, epochs=500, batch_size=32)
    nn.save("model.pkl")
    
    # Evaluate
    print(f"\nTest Accuracy: {nn.accuracy(X_test, y_test):.4f}")

    # Make Predictions
    predictions = nn.predict(X_test[:5])
    probabilities = nn.predict_proba(X_test[:5])
    print(f"\nSample Predictions: {predictions}")
    print(f"Sample probabilities:\n{probabilities}")