TDM 30200: Project 12 - Large Language Models (LLMs): Sequence Modeling

def read_lines(file_path, n, start = 0):
    lines = []
    with open(file_path, 'r') as f: # open the file in read mode
        for i, line in enumerate(f):
            if i < start:
                continue
            lines.append(line.strip())
            if len(lines) == n:
                break
    return lines

import codecs
def clean_text(text):
    #modify the text here
    # 1) decode escape sequences
    # 2) remove the starting and ending double quotes
    # 3) replace any newlines with a space
    # 4) return the cleaned text

    text = codecs.decode(text, 'unicode_escape')
    text = text.replace('\n', ' ')
    text = text.lower()
    text = ''.join(c for c in text if c.isalpha() or c.isspace())


    return text

def get_next_word_loop(start_string, n=10):
    # generate the next n words and output the final string
    for _ in range(n):
        # get the next word using the n-gram model
        # your code here

        # append the next word to the start_string, with a space in between
        # your code here

    # return the final string
    # your code here

import numpy as np
my_ngram = NGram(3, is_character_based=False)
my_ngram.set_data(read_lines('music.txt', 50000))

print(get_next_word_loop("this is a",5))

print(get_next_word_loop("this is a",20))

from gensim.models import FastText, KeyedVectors
def generate_fasttext_vectors(lines, filename):
    # Clean the text using the clean_text function
    cleaned_lines = [clean_text(line) for line in lines]

    # Split each line into words using .split()
    tokenized_lines = [line.split() for line in cleaned_lines]

    # Create a Word2Vec model using the tokenized lines
    model = FastText(tokenized_lines, vector_size=100, window=5, min_count=1, workers=1)

    # Save the model to disk
    model.save(f'{filename}.model')

    # Save the word vectors to disk
    word_vectors = model.wv
    word_vectors.save(f'{filename}.wordvectors')

    return word_vectors

keyedvectors = generate_fasttext_vectors(read_lines('/anvil/projects/tdm/data/amazon/music.txt', 5000, 0), 'P12_fasttext')

def create_dataset(data, word_embeddings, n=3):
    input_data = []
    output_data = []

    for line in data:
        # clean the line
        # your code here

        # split the line into words
        words = []
        # your code here

        # if the number of words is less than n, skip the line
        if len(words) < n:
            continue

        # convert the list of words into a list of word embeddings
        embeddings = []
        # your code here

        # group the embeddings into groups of n consecutive words
        embedding_groups = []
        # your code here

        r_i = []
        r_o = []
        # append a numpy array of the sequence word embeddings to the input data list and the next word embeddings to the output data list
        for group in embedding_groups:
            # your code here


        # append a numpy array of r_i to input_data, and r_o to output_data
        # your code here

    # return the list of words, embeddings, input_data, and output_data
    return (words, embeddings, input_data, output_data)

words, embeddings, input_dataset, output_data = create_dataset(read_lines('music.txt', 1500), keyedvectors, n=3)

print(input_dataset[0].shape) # (4, 2, 100), we have 4 sequences of 2 words, each with 100 dimensions
print(input_dataset[1].shape) # (2, 2, 100), we have 2 sequences of 2 words, each with 100 dimensions
print(input_dataset[2].shape) # (39, 2, 100), we have 39 sequences of 2 words, each with 100 dimensions
print(len(input_dataset)) # 1360, there are 1360 reviews that were long enough to be used
print(len(output_data)) # 1360, there are 1360 reviews that were long enough to be used

print(output_data[0][0][:5]) # [ 0.88376546  0.7545694  -0.9747805  -0.74862236  0.05938531]

print(input_dataset[10][0][0][:5]) # [-0.1334532   0.10588685 -2.6614566  -1.1856614   0.5889898 ]

print(output_data[8][0][:5]) # [ 0.4304928   0.8276741  -0.91861534 -0.3446058   0.46729067]

def cosine_similarity_vec(v1, v2):
    # find the cosine similarity between two vectors
    dot_product = np.dot(v1, v2)
    magnitude_v1 = np.linalg.norm(v1)
    magnitude_v2 = np.linalg.norm(v2)

    return dot_product / (magnitude_v1 * magnitude_v2)

def get_word_from_embedding(embedding, keyedvectors):
    # get a key: value pair of all the word embeddings in the keyedvectors object
    # your code here

    # for each word embedding in the keyedvectors object, calculate the cosine similarity to the given word embedding
    # your code here

    # find the word with the highest cosine similarity to the given word embedding
    # your code here

    # return the word with the highest cosine similarity
    # your code here

print(words[0:10]) # ['great', 'video', 'you', 'are', 'taught', 'the', 'tricks', 'of', 'the', 'trade']
print([get_word_from_embedding(embeddings[i], keyedvectors) for i in range(10)])
# [('great', 0.9999999), ('video', 1.0), ('you', 1.0000001), ('are', 1.0000001), ('taught', 1.0), ('the', 1.0), ('tricks', 0.9999999), ('of', 1.0000001), ('the', 1.0), ('trade', 0.99999994)]

print(get_word_from_embedding(input_dataset[5][10][0], keyedvectors)) # should print ('still', 1.0)
print(get_word_from_embedding(input_dataset[5][11][0], keyedvectors)) # should print the word ('sounding', 0.99999994)
print(get_word_from_embedding(input_dataset[110][5][0], keyedvectors)) # should print the word ('rock', 1.0000001)

from torch import nn
from torch import optim
import torch
import time
class RNNModel(nn.Module):
    def __init__(self, input_size=100, hidden_size=128, output_size=100, input_sequence_length = 2): # our word embeddings are 100 dimensions, and we will use a hidden size of 128.
        super().__init__() # initialize the base class
        self.input_sequence_length = input_sequence_length # this is our input_sequence_length. We won't use it in the model, but you will use it to write your predict function in the next question.
        self.rnn = nn.RNN(input_size, hidden_size, batch_first=True) # create the RNN layer, using the input size and a hidden size. The batch_first = True parameter means that the input data will be in the format (batch_size, sequence_length, input_size). This is important because we will be using batches of data to train the model. Batches are used to essentially group data together to speed up training, but in this case we can use it to group our sequences together with a batch size of 1.
        self.fc = nn.Linear(hidden_size, output_size) # add a linear layer to the model, which will take the hidden layer and output the next word embedding. The output size is the same as the input size, because we want to output a word embedding that is the same size as the input word embedding.

    def forward(self, x, hidden): # this will take in the current input data and the hidden state, and return the output data and the updated hidden state.
        out, hidden = self.rnn(x, hidden) # get the output of the RNN layer, and update the hidden state
        out = self.fc(out[:, -1, :]) # pass the output of the RNN layer through the linear layer. The out[:, -1, :] means that we are only taking the last output of the RNN layer. Although this is only the last word from the sequence, since it has been passed through the RNN layer along with the hidden state the entire sequence should be encoded here.
        return out, hidden

    def train(self, input_data, output_data, num_epochs=5, learning_rate=0.001): # this function will train the model using our input and output data from question 2. Additionally, we can specify how many epochs to run and the learning rate of the model.

        criterion = nn.MSELoss() # use Mean Squared Error as the criterion
        optimizer = optim.Adam(self.parameters(), lr=learning_rate) # use Adaptive Moment Estimation (Adam) as the optimizer.

        # for each epoch
        for epoch in range(num_epochs):
            # set the loss of the epoch to 0
            total_loss = 0

            # for each review (pair of input and output data arrays for the review)
            s = time.time()

            for i, (r_i, r_o) in enumerate(zip(input_data, output_data)): # iterate through each review we have

                if i % 1000 == 0: # occasionally print the progress of the training, along with the elapsed time
                    print(f'Processing review {i+1}/{len(input_data)}')
                    print(f'Elapsed time: {time.time() - s} seconds')
                    s = time.time()

                # set `hidden_state` variable to None each review, as we want to reset the hidden state for each review
                '''YOUR CODE HERE'''

                # for each sequence and output pair in the review
                for seq, o in zip(r_i, r_o):

                    # convert the sequence and output to tensors to be passed to the model
                    input_tensor = torch.tensor(seq, dtype=torch.float32).unsqueeze(1) # dimensions [2, 1, 100]
                    output_tensor = torch.tensor(o[0], dtype=torch.float32) # dimensions [100]

                    # set the optimizer to zero gradients using it's `zero_grad` function
                    '''YOUR CODE HERE'''

                    # perform a forward pass of the model using the input tensor and hidden state, and get the output and updated hidden state
                    output, hidden = # your code here

                    output = output[-1, 0] # get the last output of the RNN layer, which is the output we want to use for the next word embedding

                    if hidden is not None:
                        hidden = hidden.detach() # We need to detach the hidden state from the graph so that it isn't updated with the gradients.

                    # calculate the loss using the criterion
                    loss = criterion(output, output_tensor)

                    # backpropogate the loss, using the `backward` function of the loss
                    '''YOUR CODE HERE'''

                    # step the optimizer, using the `step` function of the optimizer
                    '''YOUR CODE HERE'''

                    # add loss.item() to the total loss for the epoch
                    '''YOUR CODE HERE'''


            # print the loss for the epoch
            print(f'Epoch {epoch+1}/{num_epochs}, Loss: {total_loss/len(input_data)}')

        return total_loss/len(input_data)

# create the model
hidden_size = 128 # size of the hidden layer
output_size = 100 # size of the output layer
input_size = 100
# input size is the size of each input data, which is n word embeddings
model = RNNModel(input_size, hidden_size, output_size)

model.train(input_dataset, output_data, num_epochs=5, learning_rate=0.001) # train the model on the input and output data

Processing review 1/1360
Elapsed time: 0.0002453327178955078 seconds
Processing review 1001/1360
Elapsed time: 43.138824224472046 seconds
Epoch 1/5, Loss: 138.86604140063682
Processing review 1/1360
Elapsed time: 1.6927719116210938e-05 seconds
Processing review 1001/1360
Elapsed time: 44.072826862335205 seconds
Epoch 2/5, Loss: 137.2912354198065
Processing review 1/1360
Elapsed time: 8.58306884765625e-06 seconds
Processing review 1001/1360
Elapsed time: 44.183839559555054 seconds
Epoch 3/5, Loss: 136.87381747121415
Processing review 1/1360
Elapsed time: 8.58306884765625e-06 seconds
Processing review 1001/1360
Elapsed time: 44.282142162323 seconds
Epoch 4/5, Loss: 136.59735957769556
Processing review 1/1360
Elapsed time: 1.0013580322265625e-05 seconds
Processing review 1001/1360
Elapsed time: 44.27771878242493 seconds
Epoch 5/5, Loss: 135.70652526225334
135.70652526225334

def predict_from_string(input_string, model, keyedvectors):
    # clean the input string using the clean_text function
    '''YOUR CODE HERE'''

    # split the input string into words using .split()
    '''YOUR CODE HERE'''

    # create a list of word embeddings for the input words using the keyedvectors object
    input_embeddings = []
    '''YOUR CODE HERE'''

    # group the embeddings into groups of model.input_sequence_length consecutive words. n-grams
    embedding_groups = []
    '''YOUR CODE HERE'''

    hidden = None
    output = None

    # we need to start passing the entire sequence of word embeddings to the model to update the hidden state
    # once we have the hidden state for the entire sequence, then we can get the next word
    for seq in embedding_groups:

        input_tensor = torch.tensor(seq, dtype=torch.float32).unsqueeze(1)
        output, hidden = model(input_tensor, hidden)

        if hidden is not None:
            hidden = hidden.detach()

    # get the predicted word embedding
    predicted_embedding = output.detach().numpy()[1]

    # get the word with the highest cosine similarity to the predicted embedding, using your `get_word_from_embedding` function
    '''YOUR CODE HERE'''

    return predicted_word, similarity

start_string = 'this is a wonderful cd and'
for i in range(7):
    newword = predict_from_string(start_string, model, keyedvectors) # this is the function you will implement in the next question
    print(newword)
    start_string += " " + newword[0]
print(start_string)

# create the model
hidden_size = 128 # size of the hidden layer
output_size = 100 # size of the output layer
input_size = 100
# input size is the size of each input data, which is n word embeddings
modellstm = LSTMModel(input_size, hidden_size, output_size)


modellstm.train(input_dataset, output_data, num_epochs=5, learning_rate=0.001) # train the model on the input and output data


start_string = 'this is a wonderful cd and'
for i in range(15):
    newword = predict_from_string(start_string, modellstm, keyedvectors)
    print(newword)
    start_string += " " + newword[0]
print(start_string)