401 Project 11 - Regression: Artificial Neural Networks (ANN) - Multilayer Perceptron (MLP)

import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
df = pd.read_csv('/anvil/projects/tdm/data/boston_housing/boston.csv')

X = df.drop(columns=['MEDV'])
y = df[['MEDV']]

scaler = StandardScaler()
X = scaler.fit_transform(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=7)

y_train = y_train.to_numpy()
y_test = y_test.to_numpy()

import numpy as np

def linear(x):
    return x

def linear_d(x):
    return np.ones_like(np.atleast_1d(x))

def relu(x):
    return np.maximum(x, 0)

def relu_d(x):
    return np.where(x>0, 1, 0)

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_d(x):
    return sigmoid(x) * (1-x)

def tanh(x):
    return np.tanh(x)

def tanh_d(x):
    return 1 - (tanh(x)**2)

def get_mse(y, y_pred):
            return np.mean((y - y_pred) ** 2)

def get_rmse(y, y_pred):
    return np.sqrt(get_mse(y,y_pred))

def get_mae(y, y_pred):
    return np.mean(np.abs(y - y_pred))

def get_r_squared(y, y_pred):
    return 1 - np.sum((y - y_pred) ** 2) / np.sum((y - np.mean(y)) ** 2)

derivative_functions = {
    'relu': relu_d,
    'sigmoid': sigmoid_d,
    'linear': linear_d,
    'tanh': tanh_d
}

activation_functions = {
    'relu': relu,
    'sigmoid': sigmoid,
    'linear': linear,
    'tanh': tanh
}

class Node:
    def __init__(self, input_size):
        # given input size (number of features for the node):
        # initialize self.weights to random values with np.random.randn
        # initialize self.bias to 0
        pass

    def forward(self, inputs):
        # calculate the dot product of the inputs and weights, add the bias, and return the result. Same as the perceptron model.
        pass

np.random.seed(11)
node = Node(3)
inputs = np.array([1, 2, 3])
output = node.forward(inputs)
print(output) # should print -0.276386648990842

class Layer:
    def __init__(self, num_nodes, input_size, activation='relu'):
        # set self.nodes to be a list of Node objects, with length num_nodes

        # check if the activation function is supported (a key in one of the provided dictionaries). if not, raise a ValueError

        # set self.activation_func and self.activation_derivative to the correct functions from the dictionaries
        pass
    def forward(self, inputs):
        # Create an list of the forward pass output of each node in the layer

        # Apply the activation function to the list of outputs and return the result
        pass

np.random.seed(11)
layer = Layer(3, 3, activation='linear')
inputs = np.array([1, 2, 3])
output = layer.forward(inputs)
print(output) # should print [-0.27638665 -3.62878191  1.35732812]

class MLP:
    def __init__(self, layer_sizes, activations):
        # we are given 'layer_sizes', a list of numbers, where each number is the number of nodes in the layer.
        # The first layer should be the number of features in the input data
        # We only need to create the hidden and output layers, as the input layer is simply our input data
        # For example, if layer_sizes = [4, 5, 2], we should set self.layers = [Layer(5, 4), Layer(2, 5)]
        # Additionally, we are given 'activations', a list of strings, where each string is the name of the activation function for the corresponding layer
        # len(activations) will always be len(layer_sizes) - 1, as the input layer does not have an activation function

        # Please set self.layers to be a list of Layer objects, with the correct number of nodes, input size, and activation function.
        pass

    def forward(self, inputs):
        # for each layer in the MLP, call the forward method with the output of the previous layer
        # then, return the final output
        pass

    def train(self, X, y, epochs=100, learning_rate=0.0001):
        for epoch in range(epochs):
            for i in range(len(X)):
                # Store the output of each layer in a list, starting with the input data
                # You should have a list that looks like [X[i], layer1_output, layer2_output, ..., outputlayer_output]

                # find the error, target value - output value


                # Now, we can perform our backpropagation to update the weights and biases of our model
                # We need to start at the last layer and work our way back to the first layer
                for j in reversed(range(len(self.layers))):
                    # get the layer object at index j

                    # get the layer_input and layer_output corresponding to the layer. Remember, self.layers does not contain the input, but outputs list above does

                    # calculate the gradient for each node in the layer
                    # same as the perceptron model, -error * activation_derivative(layer_output).
                    # However, this time it is a vector, as we are calculating the activation_derivative for everything in the layer at once


                    # Now, we must update the error for the next layer.
                    # This is so that we can calculate the gradient for the next layer
                    # This is done by taking the dot product of our gradients by the weights of each node in the current layer

                    # Finally, we can update the weights and biases of each node in the current layer
                    # Remember, our gradient is a list, so each node in the layer will have its own corresponding gradient
                    # Otherwise, the process is the same as the perceptron model.
                    for k, node in enumerate(layer.nodes):
                        # update the weights and bias of the node
                        pass

    def test(self, X, y, methods=['mse', 'rmse', 'mae', 'r_squared']):
        # Calculate metrics for each method
        # First, get the predictions for each input in X

        # Then, for each method the user wants, call the corresponding function with input y and predictions

        # Finally return a dictionary with the metric as key and the result as value

        pass

np.random.seed(11)
mlp = MLP([3, 4, 2], ['relu', 'linear'])
inputs = np.array([1, 2, 3])
output = mlp.forward(inputs)
print(output) # should print [-1.77205771 -0.04217909]

X = np.array([[1, 2, 3], [4, 5, 6]])
y = np.array([[0, 1], [1, 0]])

metrics = mlp.test(X, y)
print(metrics) # should print {'mse': 2.698031745867772, 'rmse': 1.6425686426654358, 'mae': 1.6083905323341714, 'r_squared': -9.792126983471087}

np.random.seed(3)
mlp = MLP([X_train.shape[1], 1], ['linear'])
mlp.layers[0].nodes[0].weights = np.zeros(X_train.shape[1])
mlp.train(X_train, y_train, epochs=100, learning_rate=0.01)
print(mlp.layers[0].nodes[0].weights) # should print the same weights as the perceptron model
print(mlp.layers[0].nodes[0].bias) # should print the same bias as the perceptron model
mlp.test(X_test, y_test) # should print the same metrics as the perceptron model

np.random.seed(3)
mlp = MLP([X_train.shape[1], 2, 3, 1], ['linear','linear','linear'])
mlp.train(X_train, y_train, epochs=1000, learning_rate=0.0001)
mlp.test(X_test, y_test) # should output {'mse': 17.78775654565155, 'rmse': 4.217553383853197, 'mae': 3.2032070058415836, 'r_squared': 0.7119015806656752}

np.random.seed(1)
initialized_mlp = MLP([80,80,80,80], ['relu','relu','relu'], ['random','xavier','he'])

original_random = initialized_mlp.layers[0].nodes[0].weights
xavier = initialized_mlp.layers[1].nodes[0].weights
he = initialized_mlp.layers[2].nodes[0].weights

import matplotlib.pyplot as plt

plt.hist(original_random, bins=50, alpha=0.5, label='Random')
plt.hist(xavier, bins=50, alpha=0.5, label='Xavier')
plt.hist(he, bins=50, alpha=0.5, label='He')
plt.legend(loc='upper right')
plt.show()