Assignment_1_NLP

!pip install nltk

# Import relevant packages import nltk from nltk import bigrams, trigrams from collections import Counter, defaultdict from textblob import TextBlob # Download corpus nltk.download('reuters') nltk.download('punkt') from nltk.corpus import reuters

# Create unigram model counts = Counter(reuters.words()) total_count = len(reuters.words()) # Total number of unigrams print('Total unigrams in reuters corpus: ', total_count) # Most common unigrams print('The 20 most common unigrams are: ') print(counts.most_common(n=20))

# Compute probabilities for word in counts: counts[word] /= float(total_count) # Check that probabilities add up to 1 print( sum(counts.values())) # The most common 20 words are ... print('The 20 most common unigrams are: ') print(counts.most_common(n=20))

# Create bigram model bigram_model = defaultdict(lambda: defaultdict(lambda: 0)) for sentence in reuters.sents(): for w1, w2 in bigrams(sentence, pad_right=True, pad_left=True): bigram_model[w1][w2] += 1 # Compute probabilities for w1 in bigram_model: total_count = float(sum(bigram_model[w1].values())) for w2 in bigram_model[w1]: bigram_model[w1][w2] /= total_count

# Test bigram model print(bigram_model["the"]["economists"]) print(bigram_model["the"]["nonexistingword"]) print(bigram_model[ None]["The"])

# Creat trigram model trigram_model = defaultdict(lambda: defaultdict(lambda: 0)) for sentence in reuters.sents(): for w1, w2, w3 in trigrams(sentence, pad_right=True, pad_left=True): trigram_model[(w1, w2)][w3] += 1 # Compute probabilities for w1_w2 in trigram_model: total_count = float(sum(trigram_model[w1_w2].values())) for w3 in trigram_model[w1_w2]: trigram_model[w1_w2][w3] /= total_count

# Test trigram model print(trigram_model["what", "the"]["economists"]) print(trigram_model["what", "the"]["nonexistingword"]) print(trigram_model[None, None]["The"])

# Naive approach: 4 sets of values lambda_set1 = [1/3.0, 1/3.0, 1/3.0] lambda_set2 = [0.2, 0.4, 0.4] lambda_set3 = [0.1, 0.1, 0.8] lambda_set4 = [0.0, 0.0, 1.0] # Interpolation strategy def smoothed_trigram_probability(lambdas, trigram): # Ensure input is a trigram assert len(trigram)==3 # Assign lambdas lambda1 = lambdas[0] lambda2 = lambdas[1] lambda3 = lambdas[2] # Extract words out of trigram u,v,w = trigram[0],trigram[1],trigram[2] # Calculate smoothed probability prob = (lambda1* counts[w])+\ (lambda2*bigram_model[v][w])+\ (lambda3*trigram_model[u,v][w]) return prob

# Test 4 different lambda values print(smoothed_trigram_probability(lambda_set1, ["what", "the","economists"])) print(smoothed_trigram_probability(lambda_set2, ["what", "the","economists"])) print(smoothed_trigram_probability(lambda_set3, ["what", "the","economists"])) print(smoothed_trigram_probability(lambda_set4, ["what", "the","economists"]))

import random

# Generate 100 words of language using frequencies of words text = [] # Loop through 100 times for i in range(100): r = random.random() accumulator = .0 # Loop through word frequency for word, freq in counts.items(): accumulator += freq # Select word that is above the frequency threshold, and append to text if accumulator >= r: text.append(word) break # Join text list with space and print print( ' '.join(text))

# The probability of a text from operator import mul from functools import reduce print('Probability of random generated text: ', reduce(mul, [counts[w] for w in text], 1.0))

# Starting words text = [None] sentence_finished = False while not sentence_finished: # Select a random probability threshold r = random.random() accumulator = .0 # Loop through word probabilities of latest word in text latest_word = text[-1] for word, prob in bigram_model[latest_word].items(): accumulator += prob # Select word that is above the probability threshold, and append to text if accumulator >= r: text.append(word) break # If latest word is equal to none, sentence is finished if text[-1] == None: sentence_finished = True # Join text list with space and print print (' '.join([t for t in text if t]))

# Starting words text = [None, None] sentence_finished = False while not sentence_finished: # Select a random probability threshold r = random.random() accumulator = .0 # Loop through word probabilities of latest words in text latest_words = tuple(text[-2:]) for word, prob in trigram_model[latest_words].items(): accumulator += prob # Select word that is above the probability threshold and append to text if accumulator >= r: text.append(word) break # If latest word is equal to none, sentence is finished if text[-1] == None: sentence_finished = True # Join text list with space and print print(' '.join([t for t in text if t]))

e1 = 'the actor gives a convincing, charismatic performance as the multifaceted' e2 = 'Spielberg gives us a visually spicy and historically accurate real life story' e3 = 'His innovative mind entertains us now and will continue to entertain generations to come' b1 = 'Unfortunately, the film has two major flaws, one in the disastrous ending' b2 = 'If director actually thought this movie was worth anything' b3 = 'His efforts seem fruitless, creates drama where drama shouldn’t be' c = [e1, e2, e3, b1, b2, b3] e = [e1, e2, e3] b = [b1, b2, b3] train_e = [[e1, '1'], [e2, '1'], [e3, '1']] train_b = [[b1, '0'], [b2, '0'], [b3, '0']] def unique_words(corpus): dictionary = {} array = [] for c in corpus: for word in c.split(): dictionary[word] = word array.append(word) return len(dictionary) print("Unique words in whole corpus:", unique_words(c)) print("Unique words in 'entertaining' corpus:", unique_words(e)) print("Unique words in 'boring' corpus:", unique_words(b)) dict_e = unique_words(e)