Assignment 4: Natural Language Processing
by William Jönsson & Matthieu Moutot - Group 43
import collections
import random
import itertools
(a) Warmup - Word frequency
10 most frequent words in each language:
# Open the file, remove commas and dots, split the text into words
def open_remove_split(file_name):
raw_text = open(file_name).read()
t1 = raw_text.replace('.', '')
t2 = t1.replace(',', '')
t3 = t2.replace(' '', '\'')
t4 = t3.replace('"','')
text_split = t4.split()
return text_split
# English (most frequent words are similar in other texts)
text_en_sv = open_remove_split('/work/dat410_europarl/europarl-v7.sv-en.lc.en')
print('English: ', collections.Counter(text_en_sv).most_common(10))
# Swedish
text_sv_en = open_remove_split('/work/dat410_europarl/europarl-v7.sv-en.lc.sv')
print('Swedish: ', collections.Counter(text_sv_en).most_common(10))
# French
text_fr_en = open_remove_split('/work/dat410_europarl/europarl-v7.fr-en.lc.fr')
print('French: ', collections.Counter(text_fr_en).most_common(10))
# German
text_de_en = open_remove_split('/work/dat410_europarl/europarl-v7.de-en.lc.de')
print('German: ', collections.Counter(text_de_en).most_common(10))
English: [('the', 19309), ('of', 9305), ('to', 8801), ('and', 6944), ('in', 6086), ('is', 4399), ('that', 4351), ('a', 4263), ('we', 3223), ('this', 3221)]
Swedish: [('att', 9181), ('och', 7038), ('i', 5950), ('det', 5687), ('som', 5028), ('för', 4959), ('av', 4013), ('är', 3840), ('en', 3724), ('vi', 3211)]
French: [('de', 14520), ('la', 9746), ('et', 6619), ("l'", 6535), ('le', 6174), ('les', 5585), ('à', 5500), ('des', 5232), ('que', 4797), ("d'", 4553)]
German: [('die', 10521), ('der', 9374), ('und', 7028), ('in', 4175), ('zu', 3167), ('den', 2976), ('wir', 2863), ('daß', 2738), ('ich', 2670), ('das', 2669)]
Probability that a randomly selected word in the text is "speaker" or "zebra":
# Function to get any word's frequency
def get_word_freq(text, word):
occ = collections.Counter(text).get(word)
if occ != None:
p_word = occ / (len(text) + occ)
return p_word
else:
return 0
p_speaker = get_word_freq(text_en_sv, 'speaker')
p_zebra = get_word_freq(text_en_sv, 'zebra')
print("Probability to get 'speaker': ", p_speaker)
print("Probability to get 'zebra': ", p_zebra)
Probability to get 'speaker': 3.893990015809599e-05
Probability to get 'zebra': 0
(b) Bigram Language Model
class LanguageHandler:
# Normalize text file
def __init__(self, file_name):
raw_text = open(file_name).read()
t1 = raw_text.replace('.', '')
t2 = t1.replace(',', '')
t3 = t2.replace(' '', '\'')
t4 = t3.replace('"','')
self.text = t4
# Get frequency of a single word
def get_word_freq(self, text, word):
if isinstance(text, str):
text = text.split()
occ = collections.Counter(text).get(word)
if occ != None:
p_word = occ / len(text)
return p_word
else:
return 0
def get_word_pair_freq(self, text, word):
if isinstance(text, str):
text = text.split()
occ = collections.Counter(text).get(word)
if occ != None:
p_word = occ / (len(text) + occ)
return p_word
else:
return 0
# Get frequency of a pair of words
def get_pair_freq(self, prev_word, word):
t1 = self.text
t2 = t1.replace(prev_word + " " + word, prev_word+word) # replace paired words by a single one
t3 = t2.split()
f_pair = self.get_word_pair_freq(t3, prev_word+word)
f_prev_word = self.get_word_freq(self.text, prev_word)
if f_prev_word == 0:
return 0
else:
return f_pair / f_prev_word
# Get frequency of a sequence of several words
def get_sequence_prob(self, raw_words):
if isinstance(raw_words, str):
words = raw_words.split()
else:
words = raw_words
prob = self.get_word_freq(self.text, words[0])
for i in range(1, len(words)):
prob *= self.get_pair_freq(words[i-1], words[i])
return prob
# Test on an example
t = LanguageHandler('/work/dat410_europarl/europarl-v7.sv-en.lc.en')
p = t.get_sequence_prob('the commission is')
p
(c) Translation Modeling
Print ten most likely translations for the word "european":
# Load and normalize english and french source texts, split them into sentences
raw_text = open('/work/dat410_europarl/europarl-v7.fr-en.lc.en').read()
t1 = raw_text.replace('.', '')
t2 = t1.replace(',', '')
t3 = t2.replace(' '', '\'')
t4 = t3.replace('"','')
english_sentences = t4.splitlines()
raw_text = open('/work/dat410_europarl/europarl-v7.fr-en.lc.fr').read()
t1 = raw_text.replace('.', '')
t2 = t1.replace(',', '')
t3 = t2.replace(' '', '\'')
t4 = t3.replace('"','')
french_sentences = t4.splitlines()
c1 = dict()
c2 = dict()
t = dict()
# Calculate delta for each iteration
def delta(i,j, f, e):
sum = 0
for e_word in e:
sum += t[f[i], e_word]
return t[f[i], e[j]] / sum
def IBM_model_1(french_sentences, english_sentences):
# Init, set random coefficients for alignment table ---------------
for index in range(len(english_sentences)):
e = english_sentences[index].split() # english sentence split into list of words
for ek in e: # for every word word in one english sentence
f = french_sentences[index].split()
for fk in f:
t[fk,ek] = random.uniform(0,1)
for T in range(5): # number of epochs (better accuracy when increasing T)
print("T:",T)
# Set pseudocounts to 0
for index in range(len(english_sentences)):
e = english_sentences[index].split() # english sentence split into list of words
for ek in e: # one word in one english sentence
c1[ek] = 0
f = french_sentences[index].split()
for fk in f:
c2[ek,fk] = 0
# Update pseudocounts
for k in range(len(english_sentences)): # senctence pair example, f ={"le chat noir} e = {"the black cat"}, pairs = (le,the), (le,black), (le,cat) ...isinstance
if k%1000 == 0:
print("k:",k)
f = french_sentences[k].split() # f is french sentence
e = english_sentences[k].split() # e its equivalent english sentence
for i in range(len(f)): # For each French word
for j in range(len(e)): # For each English word
d = delta(i, j, f, e)
c2[e[j], f[i]] += d
c1[e[j]] += d
# Recalculate coefficients in alignment table
for k in range(len(english_sentences)): # sentence pair example, f ={"le chat noir} e = {"the black cat"}, pairs = (le,the), (le,black), (le,cat) ...isinstance
f = french_sentences[k].split() # f is a french sentence
e = english_sentences[k].split() # e its equivalent english sentence
for i in range(len(f)): # For each French word
for j in range(len(e)): # For each English word
t[f[i], e[j]] = c2[e[j], f[i]] / c1[e[j]]
print(T)
return t
table = IBM_model_1(french_sentences, english_sentences)
T: 0
k: 0
k: 1000
k: 2000
k: 3000
k: 4000
k: 5000
k: 6000
k: 7000
k: 8000
k: 9000
0
T: 1
k: 0
k: 1000
k: 2000
k: 3000
k: 4000
k: 5000
k: 6000
k: 7000
k: 8000
k: 9000
1
T: 2
k: 0
k: 1000
k: 2000
k: 3000
k: 4000
k: 5000
k: 6000
k: 7000
k: 8000
k: 9000
2
T: 3
k: 0
k: 1000
k: 2000
k: 3000
k: 4000
k: 5000
k: 6000
k: 7000
k: 8000
k: 9000
3
T: 4
k: 0
k: 1000
k: 2000
k: 3000
k: 4000
k: 5000
k: 6000
k: 7000
k: 8000
k: 9000
4
table['la', 'the']
# Find 10 most likely translations of the word "european"
keys = table.keys()
keys = sorted(keys, key=lambda x:x[1])
has_found_european = False
european = dict()
for k in keys:
if k[1] == 'european':
has_found_european = True
european[k[0]] = table[k]
elif has_found_european:
break
sorted_tuples = sorted(european.items(), key=lambda item: item[1], reverse=True)
sorted_tuples[:10]
(d) Decoding
# Get three most likely translations for a given word
def find_highest_prob_translation_word(t_table, word):
keys = table.keys()
keys = sorted(keys, key=lambda x:x[0])
has_found_word = False
translations = dict()
for k in keys: # (french, english)
if k[0] == word: # french == word
has_found_word = True
translations[k[1]] = table[k] # trans[english] = table prob
elif has_found_word:
break
sorted_tuples = sorted(translations.items(), key=lambda item: item[1], reverse=True)
first_tuple_elements = [a_tuple[0] for a_tuple in sorted_tuples[:3]]
return first_tuple_elements
First use EM table to get the translated words, then use the language modelling to find the best sequence of those words.
# Get top three translation for every word in the french sentence
french_to_be_translated = 'semaine dernière'
f_words = french_to_be_translated.split()
translated_words = list()
for word in f_words:
translated_words.append(find_highest_prob_translation_word(table, word))
print(translated_words)
[['week', "week'", 'chaotic'], ['last', 'chaotic', 'cancellations']]
# Get best translation out of all possibilities (depending on order and word translation)
l = LanguageHandler('/work/dat410_europarl/europarl-v7.fr-en.lc.en')
max_prob = 0
best_translation = ''
for perm_order in itertools.permutations(translated_words):
for sentence in list(itertools.product(*perm_order)):
string = ' '.join(sentence)
p = l.get_sequence_prob(string)
if p > max_prob:
max_prob = p
best_translation = string
print('The best translation of "', french_to_be_translated, '" is: ', best_translation)
The best translation of " semaine dernière " is: last week
# Used to get data in a file
import pickle
dictionary_data = table
a_file = open("data.pkl", "wb")
pickle.dump(dictionary_data, a_file)
a_file.close()
a_file = open("data.pkl", "rb")
output = pickle.load(a_file)
print(output)
IOPub data rate exceeded.
The notebook server will temporarily stop sending output
to the client in order to avoid crashing it.
To change this limit, set the config variable
`--NotebookApp.iopub_data_rate_limit`.
Current values:
NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)
NotebookApp.rate_limit_window=3.0 (secs)