Question 1
Emil Petterson S141550
&
Laust Dixen Munck S119059
import nltk
import random
from collections import Counter, defaultdict
from nltk . corpus import PlaintextCorpusReader
from nltk import bigrams
from nltk import trigrams
from nltk.util import ngrams
from nltk.tokenize import sent_tokenize, word_tokenize
nltk.download('punkt')
nltk.download('gutenberg')
emma_sents = nltk.corpus.gutenberg.sents('austen-emma.txt')
emma_words = nltk.corpus.gutenberg.words('austen-emma.txt')
Unigram
#All of the models include characters like ',' and '.' and the do differentiate from capitalization
counts_word = Counter(emma_words)
#ounts_sent = Counter(emma_sents)
len(counts_word)
counts_word.most_common(n=10)
# Checking the most common words
counts_word.most_common(n=10)
Then the words'/characters' share of the total amount of words i calculated and then added:
for word in counts_word:
counts_word[word] /= len(counts_word)
print(counts_word.most_common(n=10))
Now a amount of words is generated using the frequencies calculated above.
The amount of words generated can be selected in the range() function.
text = []
for i in range(100):
r = random.random()
accumalator = .0
for word, freq in counts_word.items():
accumalator += freq
if accumalator >= r:
text.append(word)
break
print(' '.join(text))
Bigram
# Creating a dictionary containing the word and then the words the comes after as well as the probability:
# {(None): { '[' : Probability , ...}, ...}
# An example of this can be seen in the output
b_model = defaultdict(lambda: defaultdict(lambda: 0))
# Creating a bigram model by assign 1 each time the two combinations of words exists in the layout described above
for sentence in emma_sents:
for w1, w2 in bigrams(sentence, pad_right = True, pad_left = True):
b_model[w1][w2] += 1
# Iterating through the model and calculating the probability of the bigrams
for w1 in b_model:
total_count = float(sum(b_model[w1].values()))
# If the total_count == 0 the pass, which means if the bigram does not exist, then it should skip it
if total_count == 0:
pass
# Again if it does not exist assign the value 0
for w2 in b_model[w1]:
if total_count == 0:
b_model[w1][w2] = 0
# If the brigram is not 0, then it calculates the probability
else:
b_model[w1][w2] /= total_count
b_model['VOLUME']
Generates a random text
text = [None]
sentence_finished = False
count = 0
while not sentence_finished:
r = random.random()
accumulator = .0
if len(text) == 1:
var = b_model[None]
else:
tup_text = tuple(text)
var = b_model[tup_text[-1]]
for word in var.keys():
accumulator += var[word]
if accumulator >= r:
text.append(word)
break
else:
pass
if text[-1:] == [None]:
sentence_finished = True
print ('\nThe generated sentence is: \n', ' '.join([t for t in text if t]))
Trigram
# # Creating a dictionary containing the two words and then the words the comes after as well as the probability:
# {(None, None): { '[' : Probability , ...}, ...}
# An example of this can be seen in the output
t_model = defaultdict(lambda: defaultdict(lambda: 0))
# Creating a trigram model by assigning 1 each time the trigram exists in the layout described above
for sentence in emma_sents:
for w1, w2, w3 in trigrams(sentence, pad_right = True, pad_left = True):
t_model[(w1, w2)][w3] += 1
# Iterating through the model and calculating the probability of the bigrams
for w1_w2 in t_model:
# If the total_count == 0 the pass, which means if the bigram does not exist, then it should skip it
total_count = float(sum(t_model[w1_w2].values()))
if total_count == 0:
pass
# Again if it does not exist assign the value 0
for w3 in t_model[w1_w2]:
if total_count == 0:
t_model[w1_w2][w3] = 0
# If the brigram is not 0, then it calculates the probability
else:
t_model[w1_w2][w3] /= total_count
t_model['the', 'child']
Generates a random text based on the trigram model
text = [None, None]
sentence_finished = False
while not sentence_finished:
r = random.random()
accumulator = .0
for word in t_model[tuple(text[-2:])].keys():
accumulator += t_model[tuple(text[-2:])][word]
if accumulator >= r:
text.append(word)
break
else:
pass
if text[-2:] == [None, None]:
sentence_finished = True
print ('\nThe generated sentence is: \n', ' '.join([t for t in text if t]))
1.2 0-probability fix
Laplace smoothing will fix the 0-probability problem which essentially adds an entry of each token once that can be found in either test set of training set. This can be seen in the below formula, which is a 1-plus smoothing function, where w corresponds to each word, V the vocabulary and C the text doucment.
1.3 Highest probability
from statistics import mean
def prob_finder(corp_sents, t_l, b_l, u_l):
prob_lst = []
for sent in corp_sents:
t_prob = 1
b_prob = 1
u_prob = 1
#Constructing word sets
t_sent_lst = trigrams(sent, pad_right = True, pad_left = True)
b_sent_lst = bigrams(sent, pad_right = True, pad_left = True)
u_sent_lst = Counter(sent)
#Constructing trigram model and adding probability
for w1, w2, w3 in t_sent_lst:
if t_model[w1,w2][w3] >0:
t_prob *= t_model[w1,w2][w3]
#Constructing bigram model and adding probability
for w1, w2 in b_sent_lst:
if b_model[w1][w2] > 0:
b_prob *= b_model[w1][w2]
#Constructing unigram model and adding probability
for w in u_sent_lst:
u_sent_lst[w] /= len(u_sent_lst)
if u_sent_lst[w] > 0:
u_prob *= u_sent_lst[w]
#Appending the probabilities to a list
ite_lst = [t_prob, b_prob, u_prob]
#Calculating over_all probability
o_prob = (t_prob*t_l) + (b_prob*b_l) + (u_prob*u_l)
prob_lst.append(o_prob)
#Calulating average probability of the models together
avg = sum(prob_lst) / len(prob_lst)
print('Average for lambdas,',t_l,b_l,u_l, ': ',avg)
prob_finder(emma_sents, 1/3,1/3,1/3)
prob_finder(emma_sents, 0.2,0.4,0.4)
prob_finder(emma_sents, 0.1,0.4,0.6)
prob_finder(emma_sents, 0.1,0.1,0.8)
prob_finder(emma_sents, 0.8,0.1,0.1)
prob_finder(emma_sents, 0.1,0.8,0.1)
prob_finder(emma_sents, 0.2,0.6,0.2)
prob_finder(emma_sents, 0.025,0.025,0.95)
The lambda order in the above function is trigram, bigram and then unigram. Running the function with different lambda values, it seems that the function creates the highest probabilities when the unigram is weighted the highest.
print(emma_sents[3])
Question 2
import nltk
import re
import numpy as np
from collections import Counter, defaultdict
from nltk . corpus import PlaintextCorpusReader
nltk.download('punkt')
# Insert training set for class 1 as one str
ent = "the actor gives a convincing, charismatic performance as the multifaceted Spielberg gives us a visually spicy and historically accurate real life story His innovative mind entertains us now and will continue to entertain generations to come"
# Insert the Nc for class 1
ent_nr_sen = 3
# Insert training set for class 2 as one str
bor = "Unfortunately, the film has two major flaws, one in the disastrous ending If director actually thought this movie was worth anything His efforts seem fruitless, creates drama where drama shouldn’t be"
# Insert the Nc for class 2
bor_nr_sen = 3
# Insert test set
test = 'film is a innovative drama, entertains, but disastrous ending'
# Splitting sets based on space
ent_dic = sorted(ent.lower().split(' '))
bor_dic = bor.lower().split(' ')
test_dic = test.lower().split(' ')
print('N ent : ', len(ent_dic),'\n\nN bor : ', len(bor_dic))
# See how many times the word occour in each training set (insert word in x =)
x = 'and'
print("\nEntertaining The word \"",x,"\" occours ",ent_dic.count(x))
print("Boring The word \"",x,"\" occours ",bor_dic.count(x),'\n')
# vocabulary for union of all words types in classes
all_dic = ent_dic + bor_dic
all_u = []
[all_u.append(x) for x in all_dic if x not in all_u]
print('all_dic len : ', len(all_dic),'\nall_u len : ', len(all_u))
tot_nr_sen = ent_nr_sen + bor_nr_sen
Now we have the following information which we need later:
Removing unwanted characters
ent_word = nltk.word_tokenize(ent)
bor_word = nltk.word_tokenize(bor)
nonPunct = re.compile('.*[A-Za-z0-9].*')
ent_fil = [w for w in ent_word if nonPunct.match(w)]
bor_fil = [w for w in bor_word if nonPunct.match(w)]
ent_word = Counter(ent_fil)
bor_word = Counter(bor_fil)
Removing words from test corpus that does not exist in either of the training sets:
test_set = list(set(all_u) & set(test_dic))
Removing words from ent and bor text corpora that does not exist in the test set and implementing add-1 smoothing
test_set_ent = list(set(test_set) & set(ent_word))
test_set_ent = test_set_ent + test_set
print(test_set_ent)
test_set_bor = list(set(test_set) & set(bor_word))
test_set_bor = test_set_bor + test_set
print(test_set_bor)
#Count each word:
count_test_set_ent = Counter(test_set_ent)
count_test_set_bor = Counter(test_set_bor)
#Convert to dict
count_test_set_ent_d = dict(count_test_set_ent)
count_test_set_bor_d = dict(count_test_set_bor)
list_ent = list(count_test_set_ent_d.values())
list_bor = list(count_test_set_bor_d.values())
print('\nEntertaintment set as dictionary:',count_test_set_ent_d)
print('\Boring set as dictionary:',count_test_set_bor_d)
print('\n\nProduct of bor: ',np.prod(list_bor))
print('\n\nProduct of ent: ',np.prod(list_ent))
As can be seen below with to examples the words 'film' and 'ending' we have calculated the likihoods using Eq. 4.14 from SLP-3 book 4, p.68 and is using add-1 smoothing.
To calculate which class the test sentence belongs to we use Eq. 4.9 from SLP-3 book 4, p.68.
The same logic is implemented in the code below:
def TestClass(a,b,c):
"""
a is the nr of sentences in class' share of the total amount of sentences (ent_nr_sen)
b is prod of sen or bor (prod_sen or prod_ent)
c is the ent_dic or bor_dic"""
return (a/tot_nr_sen) * (np.prod(b)/(len(all_u)+len(c))**len(b))
P_ent = TestClass(ent_nr_sen, list_ent, ent_dic)
P_bor = TestClass(bor_nr_sen, list_bor, bor_dic)
print('P(ent) = ', P_ent, '\nP(bor) = ', P_bor,'\n')
if P_ent > P_bor:
print('The model predicts that the class Entertaintment for the test sentence with a probability of : ', P_ent)
elif P_ent == P_bor:
print('The model predicts equal probability that the test sentence belongs to either of the class')
else:
print('The model predicts that the class Boring for the test sentence with a probability of : ', P_bor)
