TDM 40200: Project 1 - Multilayer Perceptron

import numpy as np
def step(x):

    output = np.where(x >= 0, 1, 0)

    return output

def sigmoid(x):

    output = 1 / (1 + np.exp(-x))

    return output

def relu(x):

    output = np.maximum(0, x)

    return output

from typing import Callable

class Perceptron:
    def __init__(self, input_size: int, learning_rate: float = 0.1, activation_function: Callable = sigmoid):
        self.weights = np.random.rand(input_size)  # Initialize weights randomly
        self.bias = np.random.rand(1)  # Initialize bias randomly
        self.learning_rate = learning_rate
        self.activation_function = activation_function

    def predict(self, x):
        # Compute the weighted sum of inputs by applying a numpy dot product between x and the weights
        weighted_sum = # YOUR CODE HERE

        # Add the bias to the weighted sum

        weighted_sum += # YOUR CODE HERE

        # Apply the activation function to the weighted sum

        output = # YOUR CODE HERE

        return output

    def train(self, X, y, epochs: int = 100):
        for epoch in range(epochs):
            for xi, yi in zip(X, y):
                # predict the output for the given input
                prediction = # YOUR CODE HERE

                # calculate the error between expected and predicted output (expected - predicted)
                error = # YOUR CODE HERE

                # multiply the error by the learning rate to get our adjustment
                adjustment = # YOUR CODE HERE

                # update the weights by adding the adjustment multiplied by the original input
                self.weights += # YOUR CODE HERE

                # update the bias by adding the adjustment
                self.bias += # YOUR CODE HERE

# Simple test example. We have 3 inputs, and we want to predict 1 if there are at least two 1s in the inputs, otherwise 0.
np.random.seed(11)  # For reproducibility
X = np.array([ [0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]])
y = np.array([0, 0, 0, 1, 0, 1, 1, 1])  # Expected output

perceptron = Perceptron(input_size=3, learning_rate=0.1, activation_function=step)
perceptron.train(X, y, epochs=100)

# Test the perceptron with the training data
for i, xi in enumerate(X):
    print(f"Input: {xi}, Predicted Output: {perceptron.predict(xi), 'Expected Output:', y[i]}")

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

torch.manual_seed(11)  # For reproducibility
np.random.seed(11)  # For reproducibility

class SingleLayerPerceptron(nn.Module):
    def __init__(self, input_size, output_size, activation_function=torch.sigmoid):
        super().__init__()
        # Define a single linear layer of perceptrons with the specified input and output sizes
        self.layer = nn.Linear(input_size, output_size)

        # Set the activation function. Torch comes with many built-in activation functions, such as torch.relu, torch.sigmoid, torch.tanh, etc. You can also define your own activation function if needed.
        self.activation_function = activation_function

    def forward(self, x):
        # We can apply a layer to the input, simply by calling it like a function
        x = # YOUR CODE HERE, apply the linear layer to the input x and store the result in x

        # Then, we can do the same with the activation function
        x = # YOUR CODE HERE, apply the activation function to x and store the result in x

        return x

# Example usage
X = np.array([ [0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]])
y = np.array([0, 0, 0, 1, 0, 1, 1, 1])  # Expected output

# We must first convert our numpy arrays to torch tensors, as PyTorch works with tensors (which are similar to numpy arrays, but have some nice features such as GPU support)
X_tensor = torch.tensor(X, dtype=torch.float32)
y_tensor = torch.tensor(y, dtype=torch.float32).view(-1, 1) # .view(-1, 1) reshapes the tensor to have a single column, as opposed to a single row.

# Define the model
input_size = X_tensor.shape[1]  # Number of features in the input
output_size = y_tensor.shape[1]  # Number of outputs (in this case, 1 for binary classification)

# define out model with the input size, output size, and the sigmoid activation function
model = # YOUR CODE HERE

# Define a loss function and an optimizer
# Binary Cross Entropy Loss, commonly used for binary classification tasks. In this case, we are using it because our output is binary (0 or 1).
loss_function = nn.BCELoss()
# Stochastic Gradient Descent optimizer with a learning rate of 0.1. As you may notice, we pass in the model's parameters to the optimizer as a reference, allowing the optimizer to update the model's weights during training.
optimizer = optim.SGD(model.parameters(), lr=0.1)

# Training loop
epochs = 100
for epoch in range(epochs):
    # Compute the model's predictions, simply by calling the model as a function, passing in the input tensor
    predictions = # YOUR CODE HERE, pass in the input tensor X_tensor to the model to get the predictions

    # Calculate the loss between the predictions and the expected output, by calling the loss function with the predictions and the expected output tensor
    loss = # YOUR CODE HERE, pass in the predictions and the expected output tensor y_tensor to the loss function

    # Zero the gradients before the backward pass by calling the optimizer's `zero_grad()` function
    # YOUR CODE HERE


    # Have the loss function calculate the backwards pass, which computes the gradients of loss with respect to the model's parameters. simply call the loss's `backward()` function
    # YOUR CODE HERE

    # Update the model's weights by calling the optimizer's `step()` function
    # YOUR CODE HERE

    # Print the loss every 10 epochs
    if epoch % 10 == 0:
        print(f'Epoch {epoch}, Loss: {loss.item()}')

# Test the perceptron with the training data
for i, xi in enumerate(X):
    print(f"Input: {xi}, Predicted Output: {perceptron.predict(xi), 'Expected Output:', y[i]}")

class MultilayerPerceptron(nn.Module):
    def __init__(self, input_size: int = 2, hidden_sizes: list = [], output_size: int = 1, activation_functions: list = [nn.Sigmoid]):
        super().__init__()

        # create a list to hold the layers
        layers = []

        # Let's start by creating our input layer. Create a linear layer with the input size of `input_size`, and the output size of either the first hidden layer size if it exists, or the output size if it does not.
        layers.append(nn.Linear(input_size, hidden_sizes[0] if hidden_sizes else output_size))

        # Then, we can add the activation function for the input layer. Use the first activation function from the `activation_functions` list.
        layers.append(activation_functions[0]())

        # If there are no hidden layers, we can stop the model here and combine our layers list into a sequential model
        if not hidden_sizes:
            self.model = nn.Sequential(*layers)
            return

        # Now that we know we have at least one hidden layer, we can loop through the hidden sizes and create a linear layer for each hidden layer size.
        # To make this easier, we will firstly append the output size to the end of the hidden sizes list, so we can loop through it and create the layers in one go.
        hidden_sizes.append(output_size)
        # Iterate through all hidden sizes except the last one.
        for i in range(len(hidden_sizes) - 1):
            # Create a linear layer with an input size of the current hidden layer size, and an output size of the next hidden layer size. Append this layer to the layers list.
            layers.append(nn.Linear(hidden_sizes[i], hidden_sizes[i + 1]))

            # Then, we can add the activation function for the current hidden layer. Use the next activation function from the `activation_functions` list. if i=0, use activation_functions[1]
            layers.append(activation_functions[i + 1]())

        # Now that we have created all the layers, we can combine them into a sequential model
        self.model = nn.Sequential(*layers)

    def forward(self, x):
        # We can apply the model to the input, simply by calling it like a function
        return self.model(x)  # Apply our sequential model to the input x

# Example usage
X = np.array([ [0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]])
y = np.array([0, 0, 0, 1, 0, 1, 1, 1])  # Expected output

# We must first convert our numpy arrays to torch tensors, as PyTorch works with tensors (which are similar to numpy arrays, but have some nice features such as GPU support)
X_tensor = torch.tensor(X, dtype=torch.float32)
y_tensor = torch.tensor(y, dtype=torch.float32).view(-1, 1) # .view(-1, 1) reshapes the tensor to have a single column, as opposed to a single row.

# Define the model
input_size = X_tensor.shape[1]  # Number of features in the input
output_size = y_tensor.shape[1]  # Number of outputs (in this case, 1 for binary classification)

# define out model with the input size, output size, and the sigmoid activation function
model = # YOUR CODE HERE

# Define a loss function and an optimizer
# Binary Cross Entropy Loss, commonly used for binary classification tasks. In this case, we are using it because our output is binary (0 or 1).
loss_function = nn.BCELoss()
# Stochastic Gradient Descent optimizer with a learning rate of 0.1. As you may notice, we pass in the model's parameters to the optimizer as a reference, allowing the optimizer to update the model's weights during training.
optimizer = optim.SGD(model.parameters(), lr=0.1)

# Training loop
epochs = 100
for epoch in range(epochs):
    # Compute the model's predictions, simply by calling the model as a function, passing in the input tensor
    predictions = # YOUR CODE HERE, pass in the input tensor X_tensor to the model to get the predictions

    # Calculate the loss between the predictions and the expected output, by calling the loss function with the predictions and the expected output tensor
    loss = # YOUR CODE HERE, pass in the predictions and the expected output tensor y_tensor to the loss function

    # Zero the gradients before the backward pass by calling the optimizer's `zero_grad()` function
    # YOUR CODE HERE


    # Have the loss function calculate the backwards pass, which computes the gradients of loss with respect to the model's parameters. simply call the loss's `backward()` function
    # YOUR CODE HERE

    # Update the model's weights by calling the optimizer's `step()` function
    # YOUR CODE HERE

    # Print the loss every 10 epochs
    if epoch % 10 == 0:
        print(f'Epoch {epoch}, Loss: {loss.item()}')

# Test the perceptron with the training data
for i, xi in enumerate(X):
    print(f"Input: {xi}, Predicted Output: {perceptron.predict(xi), 'Expected Output:', y[i]}")

import pandas as pd
train_data = pd.read_csv('/anvil/projects/tdm/data/mnist/mnist_train.csv')
test_data = pd.read_csv('/anvil/projects/tdm/data/mnist/mnist_test.csv')

# convert these rows into 2D image arrays to display, and a dataframe of labels
train_images = train_data.iloc[:, 1:].values.reshape(-1, 28, 28)
train_labels = train_data.iloc[:, 0].values

# we can use matplotlib to visualize the images
import matplotlib.pyplot as plt
plt.imshow(train_images[0], cmap='gray')
plt.title(f'Label: {train_labels[0]}')
plt.show()

import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim

train = pd.read_csv('/anvil/projects/tdm/data/mnist/mnist_train.csv')
test = pd.read_csv('/anvil/projects/tdm/data/mnist/mnist_test.csv')

X_train = torch.tensor(train.iloc[:, 1:].values, dtype=torch.float32) / 255.0  # Normalize pixel values to [0, 1] and convert to tensor
y_train = torch.tensor(train.iloc[:, 0].values, dtype=torch.long)  # Convert labels to tensor
X_test = torch.tensor(test.iloc[:, 1:].values, dtype=torch.float32) / 255.0  # Normalize pixel values to [0, 1]
y_test = torch.tensor(test.iloc[:, 0].values, dtype=torch.long)  # Convert labels to tensor

model = # YOUR CODE HERE

loss_function = nn.CrossEntropyLoss()  # CrossEntropyLoss is commonly used for multi-class classification tasks
optimizer = optim.Adam(model.parameters(), lr=0.001)  # Adam optimizer with a learning rate of 0.001

# Set the model to evaluation mode
model.eval()
# Disable gradient calculation for evaluation
with torch.no_grad():
    # Get the model's predictions on the test data
    test_predictions = model(X_test)
    # Calculate the loss on the test data
    test_loss = loss_function(test_predictions, y_test)
    # Get the predicted labels by taking the index of the maximum value in each row
    _, predicted_labels = torch.max(test_predictions, 1)
    # Calculate the accuracy by comparing the predicted labels to
    accuracy = (predicted_labels == y_test).float().mean()

import matplotlib.pyplot as plt
import numpy as np
# Our input layer takes in 784 inputs (28x28 pixels), and passes each of these 784 inputs to 128 neurons in the first hidden layer. This means we have 128 neurons, each with their own set of 784 weights.
# Get the weights of the first hidden layer.
input_weights = model.model[0].weight.data.numpy()

# Reshape the weights to 128 images, each 28x28 pixels
input_weights = input_weights.reshape(128, 28, 28)
# Display the first weights as a grayscale image
for i in range(3):  # Display the first 3 neurons' weights
    plt.figure(figsize=(4, 4))
    plt.imshow(input_weights[i], cmap='gray')
    plt.title(f'Weights of Neuron {i+1}')
    plt.axis('off')
    plt.show()

# Calculate the average weights across all neurons
average_weights = np.mean(input_weights, axis=0)
# Display the average weights as a grayscale image
plt.figure(figsize=(4, 4))
plt.imshow(average_weights, cmap='gray')
plt.title('Average Weights Across All Neurons')
plt.axis('off')
plt.show()