import numpy as np
import pandas as pd
import json
def json_to_dataframe(file):
f = open (file , "r")
data = json.loads(f.read())#Loading the json file.
#Creating empty lists to store values.
con = []
Que = []
Txt = []
for i in range(len(data['data'])): #Root tag of the json file contains 'title' tag & 'paragraphs' list.
title = data['data'][i]['title']
for p in range(len(data['data'][i]['paragraphs'])): # 'paragraphs' list contains 'context' tag & 'qas' list.
context = data['data'][i]['paragraphs'][p]['context']
for q in range(len(data['data'][i]['paragraphs'][p]['qas'])): # 'qas' list contains 'question', 'Id' tag & 'answers' list.
question = data['data'][i]['paragraphs'][p]['qas'][q]['question']
Id = data['data'][i]['paragraphs'][p]['qas'][q]['id']
is_impossible = data['data'][i]['paragraphs'][p]['qas'][q]['is_impossible']
for a in range(len(data['data'][i]['paragraphs'][p]['qas'][q]['answers'])): # 'answers' list contains 'ans_start', 'text' tags.
text = data['data'][i]['paragraphs'][p]['qas'][q]['answers'][a]['text']
con.append(context)
Que.append(question)
Txt.append(text)
new_df = pd.DataFrame(columns=['Context','Question','Answer']) # Creating empty DataFrame.
new_df.Context = con
new_df.Question = Que
new_df.Answer = Txt
print('Done')
final_df = new_df.drop_duplicates(keep='first') # Dropping duplicate rows from the create Dataframe.
return final_df
train = json_to_dataframe('train-v2.0.json')
test = json_to_dataframe('dev-v2.0.json')
Done
Done
train
test = test.reset_index(drop=True, inplace=False)
test
from keras.preprocessing.text import text_to_word_sequence
def get_start_end_words(row):
answer = text_to_word_sequence(row['Answer'])
context = text_to_word_sequence(row['Context'])
start_word=end_word=-1
match=False
if not answer:
row['Start_Word'] = start_word
row['End_Word'] = end_word
return row
for j in range(len(context)-len(answer)):
if context[j] == answer[0]:
match=True
k=0
for k in range(1, len(answer)):
if context[j+k] != answer[k]:
match=False
if match==True:
start_word=j
end_word=j+k
break
row['Start_Word'] = start_word
row['End_Word'] = end_word
return row
train = train.apply(get_start_end_words, axis=1)
test = test.apply(get_start_end_words, axis=1)
train
train = train[train['Start_Word']!=-1]
test = test[test['Start_Word']!=-1]
train = train.reset_index(drop=True, inplace=False)
test = test.reset_index(drop=True, inplace=False)
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
tokenizer = Tokenizer()
tokenizer.fit_on_texts(train.Context)
tokenizer.fit_on_texts(train.Question)
tokenizer.fit_on_texts(test.Context)
tokenizer.fit_on_texts(test.Question)
vocab_size = len(tokenizer.word_index) + 1
vocab_size
#train
context_sequences = tokenizer.texts_to_sequences(train.Context)
question_sequences = tokenizer.texts_to_sequences(train.Question)
#test
test_context_sequences = tokenizer.texts_to_sequences(test.Context)
test_question_sequences = tokenizer.texts_to_sequences(test.Question)
#train
train["Context_Sequences"] = context_sequences
train["Question_Sequences"] = question_sequences
#test
test["Context_Sequences"] = test_context_sequences
test["Question_Sequences"] = test_question_sequences
#train
context_padded = pad_sequences(context_sequences, maxlen = 700, padding="post")
question_padded = pad_sequences(question_sequences, maxlen = 50, padding="post")
#test
test_context_padded = pad_sequences(test_context_sequences, maxlen = 700, padding="post")
test_question_padded = pad_sequences(test_question_sequences, maxlen = 50, padding="post")
#train
train["Context_Sequences"] = context_padded.tolist()
train["Question_Sequences"] = question_padded.tolist()
#test
test["Context_Sequences"] = test_context_padded.tolist()
test["Question_Sequences"] = test_question_padded.tolist()
train
max_length_context = len(train.Context_Sequences[0])
print(max_length_context)
max_length_question = len(train.Question_Sequences[0])
print(max_length_question)
700
50
X_train = train[['Context_Sequences','Question_Sequences']]
y_train = train[['Start_Word','End_Word']]
X_test = test[['Context_Sequences','Question_Sequences']]
y_test = test[['Start_Word','End_Word']]
X_train
y_train
embeddings_index = dict()
f = open('glove.6B.100d.txt')
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))
Loaded 400000 word vectors.
count = 0
embedding_matrix = np.zeros((vocab_size, 100))
for word, i in tokenizer.word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
count += 1
embedding_matrix[i] = embedding_vector
print("Percentage of words covered by Glove vectors:", count/len(tokenizer.word_index)*100)
Percentage of words covered by Glove vectors: 71.92398449351087
vocab_size = len(tokenizer.word_index) + 1
embedding_vector_length = 100
max_span_begin = np.amax(y_train.Start_Word)
max_span_end = np.amax(y_train.End_Word)
batch = 32
# slice of data to be used as one epoch training on full data is expensive
slce = 10000
from tensorflow.keras import Sequential, Model, Input
from tensorflow.keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, MultiHeadAttention, concatenate, Flatten, Activation, dot
context_input = Input(shape=(max_length_context, ), dtype='int32', name='context_input')
x = Embedding(input_dim=vocab_size, output_dim=100, weights=[embedding_matrix],
input_length=max_length_context, trainable=False)(context_input)
context_LSTM = Bidirectional(LSTM(100, return_sequences=True, dropout=0.2), merge_mode='concat')(x)
question_input = Input(shape=(max_length_question, ), dtype='int32', name='ques_input')
x = Embedding(input_dim=vocab_size, output_dim=100, weights=[embedding_matrix],
input_length=max_length_question, trainable=False)(question_input)
question_LSTM, forward_h, forward_c, backward_h, backward_c = Bidirectional(LSTM(100, return_sequences=True, return_state=True, dropout=0.2), merge_mode='concat')(x)
# /!\ We are not sure if add is the right approach or should be concat
question_hidden_states = concatenate([forward_h, backward_h])
#ATTENTION LAYER
import tensorflow as tf
attention_scores = dot([context_LSTM, question_LSTM], axes=2)
attention_distribution = tf.nn.softmax(attention_scores, axis=2)
#attention output is also the question vector
attention_output = dot([attention_distribution, question_LSTM],axes=[2,1])
question_to_context = concatenate([context_LSTM, attention_output])
#We used tanh activation because most of time tanh is quickly converge than sigmoid and performs better accuracy
attention_vector = Dense(200, use_bias=False, activation='tanh')(question_to_context)
#START WORD PREDICTION
start = Dense(200)(question_hidden_states)
#We use expand_dims from tensorflow to use the start vectors as a matrix to multiply the start vector with the attention vector
start_matrix = tf.expand_dims(start, 2)
#Squeeze removes dimensions of size 1 from the shape of a tensor we added before
start_word = tf.squeeze(tf.matmul(attention_vector, start_matrix), 2)
start_word = tf.nn.softmax(start_word, axis=1,name = "Start_Word_Prediction")
#END WORD PREDICTION
end = Dense(200)(question_hidden_states)
end_matrix = tf.expand_dims(end, 2)
end_word = tf.squeeze(tf.matmul(attention_vector, end_matrix), 2)
end_word = tf.nn.softmax(end_word, axis=1,name = "End_Word_Prediction")
#ATTEMPT FAIL
def compute_loss(start, end):
J = tf.math.reduce_sum(tf.cast(start, tf.float64)) - tf.math.log(tf.cast(start, tf.float64)) + tf.math.reduce_sum(tf.cast(end, tf.float64)) - tf.math.log(tf.cast(end, tf.float64))
return J
#DEFINE THE MODEL
model = Model(inputs=[context_input, question_input], outputs=[start_word, end_word])
model.compile(optimizer='Adam', loss="sparse_categorical_crossentropy", metrics=['acc'])
model.summary()
Model: "model_92"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
context_input (InputLayer) [(None, 700)] 0
__________________________________________________________________________________________________
ques_input (InputLayer) [(None, 50)] 0
__________________________________________________________________________________________________
embedding_14 (Embedding) (None, 700, 100) 9492900 context_input[0][0]
__________________________________________________________________________________________________
embedding_15 (Embedding) (None, 50, 100) 9492900 ques_input[0][0]
__________________________________________________________________________________________________
bidirectional_14 (Bidirectional (None, 700, 400) 481600 embedding_14[0][0]
__________________________________________________________________________________________________
bidirectional_15 (Bidirectional [(None, 50, 400), (N 481600 embedding_15[0][0]
__________________________________________________________________________________________________
dot_17 (Dot) (None, 700, 50) 0 bidirectional_14[0][0]
bidirectional_15[0][0]
__________________________________________________________________________________________________
tf.nn.softmax_26 (TFOpLambda) (None, 700, 50) 0 dot_17[0][0]
__________________________________________________________________________________________________
dot_18 (Dot) (None, 700, 400) 0 tf.nn.softmax_26[0][0]
bidirectional_15[0][0]
__________________________________________________________________________________________________
concatenate_15 (Concatenate) (None, 400) 0 bidirectional_15[0][1]
bidirectional_15[0][3]
__________________________________________________________________________________________________
concatenate_16 (Concatenate) (None, 700, 800) 0 bidirectional_14[0][0]
dot_18[0][0]
__________________________________________________________________________________________________
dense_30 (Dense) (None, 400) 160400 concatenate_15[0][0]
__________________________________________________________________________________________________
dense_31 (Dense) (None, 400) 160400 concatenate_15[0][0]
__________________________________________________________________________________________________
dense_27 (Dense) (None, 700, 400) 320000 concatenate_16[0][0]
__________________________________________________________________________________________________
tf.expand_dims_20 (TFOpLambda) (None, 400, 1) 0 dense_30[0][0]
__________________________________________________________________________________________________
tf.expand_dims_21 (TFOpLambda) (None, 400, 1) 0 dense_31[0][0]
__________________________________________________________________________________________________
tf.linalg.matmul_20 (TFOpLambda (None, 700, 1) 0 dense_27[0][0]
tf.expand_dims_20[0][0]
__________________________________________________________________________________________________
tf.linalg.matmul_21 (TFOpLambda (None, 700, 1) 0 dense_27[0][0]
tf.expand_dims_21[0][0]
__________________________________________________________________________________________________
tf.compat.v1.squeeze_20 (TFOpLa (None, 700) 0 tf.linalg.matmul_20[0][0]
__________________________________________________________________________________________________
tf.compat.v1.squeeze_21 (TFOpLa (None, 700) 0 tf.linalg.matmul_21[0][0]
__________________________________________________________________________________________________
tf.nn.softmax_29 (TFOpLambda) (None, 700) 0 tf.compat.v1.squeeze_20[0][0]
__________________________________________________________________________________________________
tf.nn.softmax_30 (TFOpLambda) (None, 700) 0 tf.compat.v1.squeeze_21[0][0]
==================================================================================================
Total params: 20,589,800
Trainable params: 1,604,000
Non-trainable params: 18,985,800
__________________________________________________________________________________________________
model_history = model.fit([context_padded, question_padded], [y_train.Start_Word, y_train.End_Word], verbose=2, batch_size=batch, epochs=20)
Epoch 1/20
2564/2564 - 241s - loss: 6.3986 - tf.nn.softmax_29_loss: 3.3135 - tf.nn.softmax_30_loss: 3.0851 - tf.nn.softmax_29_acc: 0.1998 - tf.nn.softmax_30_acc: 0.2165
Epoch 2/20
2564/2564 - 236s - loss: 5.3245 - tf.nn.softmax_29_loss: 2.7753 - tf.nn.softmax_30_loss: 2.5492 - tf.nn.softmax_29_acc: 0.2919 - tf.nn.softmax_30_acc: 0.3112
Epoch 3/20
2564/2564 - 237s - loss: 4.8331 - tf.nn.softmax_29_loss: 2.5187 - tf.nn.softmax_30_loss: 2.3145 - tf.nn.softmax_29_acc: 0.3447 - tf.nn.softmax_30_acc: 0.3645
Epoch 4/20
2564/2564 - 237s - loss: 4.4192 - tf.nn.softmax_29_loss: 2.3066 - tf.nn.softmax_30_loss: 2.1127 - tf.nn.softmax_29_acc: 0.3862 - tf.nn.softmax_30_acc: 0.4071
Epoch 5/20
2564/2564 - 236s - loss: 4.1075 - tf.nn.softmax_29_loss: 2.1445 - tf.nn.softmax_30_loss: 1.9629 - tf.nn.softmax_29_acc: 0.4193 - tf.nn.softmax_30_acc: 0.4398
Epoch 6/20
2564/2564 - 236s - loss: 3.8292 - tf.nn.softmax_29_loss: 2.0017 - tf.nn.softmax_30_loss: 1.8275 - tf.nn.softmax_29_acc: 0.4519 - tf.nn.softmax_30_acc: 0.4719
Epoch 7/20
2564/2564 - 236s - loss: 3.5762 - tf.nn.softmax_29_loss: 1.8750 - tf.nn.softmax_30_loss: 1.7012 - tf.nn.softmax_29_acc: 0.4791 - tf.nn.softmax_30_acc: 0.5036
Epoch 8/20
2564/2564 - 237s - loss: 3.3402 - tf.nn.softmax_29_loss: 1.7540 - tf.nn.softmax_30_loss: 1.5863 - tf.nn.softmax_29_acc: 0.5072 - tf.nn.softmax_30_acc: 0.5326
Epoch 9/20
2564/2564 - 239s - loss: 3.1493 - tf.nn.softmax_29_loss: 1.6532 - tf.nn.softmax_30_loss: 1.4961 - tf.nn.softmax_29_acc: 0.5308 - tf.nn.softmax_30_acc: 0.5548
Epoch 10/20
2564/2564 - 241s - loss: 2.9696 - tf.nn.softmax_29_loss: 1.5632 - tf.nn.softmax_30_loss: 1.4064 - tf.nn.softmax_29_acc: 0.5503 - tf.nn.softmax_30_acc: 0.5770
Epoch 11/20
2564/2564 - 242s - loss: 2.8259 - tf.nn.softmax_29_loss: 1.4870 - tf.nn.softmax_30_loss: 1.3389 - tf.nn.softmax_29_acc: 0.5697 - tf.nn.softmax_30_acc: 0.5951
Epoch 12/20
2564/2564 - 242s - loss: 2.6817 - tf.nn.softmax_29_loss: 1.4133 - tf.nn.softmax_30_loss: 1.2684 - tf.nn.softmax_29_acc: 0.5861 - tf.nn.softmax_30_acc: 0.6129
Epoch 13/20
2564/2564 - 242s - loss: 2.5705 - tf.nn.softmax_29_loss: 1.3540 - tf.nn.softmax_30_loss: 1.2165 - tf.nn.softmax_29_acc: 0.6012 - tf.nn.softmax_30_acc: 0.6264
Epoch 14/20
2564/2564 - 242s - loss: 2.4669 - tf.nn.softmax_29_loss: 1.3032 - tf.nn.softmax_30_loss: 1.1637 - tf.nn.softmax_29_acc: 0.6139 - tf.nn.softmax_30_acc: 0.6393
Epoch 15/20
2564/2564 - 242s - loss: 2.3665 - tf.nn.softmax_29_loss: 1.2487 - tf.nn.softmax_30_loss: 1.1178 - tf.nn.softmax_29_acc: 0.6273 - tf.nn.softmax_30_acc: 0.6528
Epoch 16/20
2564/2564 - 242s - loss: 2.2938 - tf.nn.softmax_29_loss: 1.2082 - tf.nn.softmax_30_loss: 1.0856 - tf.nn.softmax_29_acc: 0.6365 - tf.nn.softmax_30_acc: 0.6611
Epoch 17/20
2564/2564 - 242s - loss: 2.2191 - tf.nn.softmax_29_loss: 1.1706 - tf.nn.softmax_30_loss: 1.0485 - tf.nn.softmax_29_acc: 0.6482 - tf.nn.softmax_30_acc: 0.6712
Epoch 18/20
2564/2564 - 242s - loss: 2.1597 - tf.nn.softmax_29_loss: 1.1402 - tf.nn.softmax_30_loss: 1.0195 - tf.nn.softmax_29_acc: 0.6534 - tf.nn.softmax_30_acc: 0.6788
Epoch 19/20
2564/2564 - 240s - loss: 2.0848 - tf.nn.softmax_29_loss: 1.0973 - tf.nn.softmax_30_loss: 0.9874 - tf.nn.softmax_29_acc: 0.6645 - tf.nn.softmax_30_acc: 0.6910
Epoch 20/20
2564/2564 - 238s - loss: 2.0305 - tf.nn.softmax_29_loss: 1.0683 - tf.nn.softmax_30_loss: 0.9622 - tf.nn.softmax_29_acc: 0.6721 - tf.nn.softmax_30_acc: 0.6959
y_pred = model.predict([test_context_padded, test_question_padded])
y_pred_start= y_pred[0].argmax(axis=-1)
y_pred_end= y_pred[1].argmax(axis=-1)
y_pred_start = pd.DataFrame(y_pred_start, columns=["Start"])
y_pred_end = pd.DataFrame(y_pred_end, columns=["End"])
predictions = pd.DataFrame(y_pred_start)
predictions["End"] = y_pred_end
predictions
from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix
d = {'Start_Word': [precision_score(y_test['Start_Word'], predictions['Start'], average="macro"), recall_score(y_test['Start_Word'], predictions['Start'], average="macro"), f1_score(y_test['Start_Word'], predictions['Start'], average="macro")],
'End_Word': [precision_score(y_test['End_Word'], predictions['End'], average="macro"), recall_score(y_test['End_Word'], predictions['End'], average="macro"), f1_score(y_test['End_Word'], predictions['End'], average="macro")]}
scores = pd.DataFrame(data=d)
scores.insert (0, "Raw", ['Precision', 'Recall', 'F1'])
scores.set_index('Raw')
scores
/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
_warn_prf(average, modifier, msg_start, len(result))
def index_to_word(context_seq, context, question, answer, word_start, word_end):
sentence = []
sentence.append(context_seq[word_start:word_end+1])
text = tokenizer.sequences_to_texts(sentence)
print("Context : ", context, "\nQuestion:", question, "\nActual Answer:", answer, "\nPredicted Answer:", text[0], "\n\n\n")
for i in range(5,25):
index_to_word(X_test.Context_Sequences[i], test.Context[i], test.Question[i], test['Answer'][i], predictions["Start"][i], predictions["End"][i])
Context : The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ("Norman" comes from "Norseman") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.
Question: What century did the Normans first gain their separate identity?
Actual Answer: 10th century
Predicted Answer: 10th and 11th centuries
Context : The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ("Norman" comes from "Norseman") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.
Question: What century did the Normans first gain their separate identity?
Actual Answer: the first half of the 10th century
Predicted Answer: 10th and 11th centuries
Context : The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ("Norman" comes from "Norseman") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.
Question: What century did the Normans first gain their separate identity?
Actual Answer: 10th
Predicted Answer: 10th and 11th centuries
Context : The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands.
Question: Who was the duke in the battle of Hastings?
Actual Answer: William the Conqueror
Predicted Answer: richard i
Context : The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands.
Question: Who ruled the duchy of Normandy
Actual Answer: Richard I
Predicted Answer: prince bohemond i
Context : The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands.
Question: What religion were the Normans
Actual Answer: Catholic
Predicted Answer:
Context : The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands.
Question: What religion were the Normans
Actual Answer: Catholic orthodoxy
Predicted Answer:
Context : The English name "Normans" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann "Northman" or directly from Old Norse Norðmaðr, Latinized variously as Nortmannus, Normannus, or Nordmannus (recorded in Medieval Latin, 9th century) to mean "Norseman, Viking".
Question: When was the Latin version of the word Norman first recorded?
Actual Answer: 9th century
Predicted Answer: 9th century
Context : In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal property. The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situated in the former Frankish kingdom of Neustria. The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions. The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine. The territory was roughly equivalent to the old province of Rouen, and reproduced the Roman administrative structure of Gallia Lugdunensis II (part of the former Gallia Lugdunensis).
Question: When was the Duchy of Normandy founded?
Actual Answer: 911
Predicted Answer: 911 as a fiefdom was established by the treaty of saint clair sur epte
Context : In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal property. The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situated in the former Frankish kingdom of Neustria. The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions. The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine. The territory was roughly equivalent to the old province of Rouen, and reproduced the Roman administrative structure of Gallia Lugdunensis II (part of the former Gallia Lugdunensis).
Question: Who did Rollo sign the treaty of Saint-Clair-sur-Epte with?
Actual Answer: King Charles III
Predicted Answer: local women and personal property the duchy of normandy which began in 911 as a fiefdom was established by the treaty of saint clair sur epte between king charles iii of west francia and the famed viking ruler rollo and was situated in the former frankish kingdom of neustria the treaty offered rollo and his men
Context : In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal property. The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situated in the former Frankish kingdom of Neustria. The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions. The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine. The territory was roughly equivalent to the old province of Rouen, and reproduced the Roman administrative structure of Gallia Lugdunensis II (part of the former Gallia Lugdunensis).
Question: What river originally bounded the Duchy
Actual Answer: Seine
Predicted Answer: river seine
Context : In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal property. The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situated in the former Frankish kingdom of Neustria. The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions. The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine. The territory was roughly equivalent to the old province of Rouen, and reproduced the Roman administrative structure of Gallia Lugdunensis II (part of the former Gallia Lugdunensis).
Question: What river originally bounded the Duchy
Actual Answer: Epte
Predicted Answer: river seine
Context : The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, blending their maternal Frankish heritage with Old Norse traditions and customs to synthesize a unique "Norman" culture in the north of France. The Norman language was forged by the adoption of the indigenous langue d'oïl branch of Romance by a Norse-speaking ruling class, and it developed into the regional language that survives today.
Question: What was the Norman religion?
Actual Answer: Catholicism
Predicted Answer: christianity
Context : The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, blending their maternal Frankish heritage with Old Norse traditions and customs to synthesize a unique "Norman" culture in the north of France. The Norman language was forged by the adoption of the indigenous langue d'oïl branch of Romance by a Norse-speaking ruling class, and it developed into the regional language that survives today.
Question: What part of France were the Normans located?
Actual Answer: north
Predicted Answer: north of france
Context : The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, blending their maternal Frankish heritage with Old Norse traditions and customs to synthesize a unique "Norman" culture in the north of France. The Norman language was forged by the adoption of the indigenous langue d'oïl branch of Romance by a Norse-speaking ruling class, and it developed into the regional language that survives today.
Question: What part of France were the Normans located?
Actual Answer: the north
Predicted Answer: north of france
Context : The Normans thereafter adopted the growing feudal doctrines of the rest of France, and worked them into a functional hierarchical system in both Normandy and in England. The new Norman rulers were culturally and ethnically distinct from the old French aristocracy, most of whom traced their lineage to Franks of the Carolingian dynasty. Most Norman knights remained poor and land-hungry, and by 1066 Normandy had been exporting fighting horsemen for more than a generation. Many Normans of Italy, France and England eventually served as avid Crusaders under the Italo-Norman prince Bohemund I and the Anglo-Norman king Richard the Lion-Heart.
Question: What was one of the Norman's major exports?
Actual Answer: fighting horsemen
Predicted Answer: 1066 normandy
Context : Soon after the Normans began to enter Italy, they entered the Byzantine Empire and then Armenia, fighting against the Pechenegs, the Bulgars, and especially the Seljuk Turks. Norman mercenaries were first encouraged to come to the south by the Lombards to act against the Byzantines, but they soon fought in Byzantine service in Sicily. They were prominent alongside Varangian and Lombard contingents in the Sicilian campaign of George Maniaces in 1038–40. There is debate whether the Normans in Greek service actually were from Norman Italy, and it now seems likely only a few came from there. It is also unknown how many of the "Franks", as the Byzantines called them, were Normans and not other Frenchmen.
Question: Who was the Normans' main enemy in Italy, the Byzantine Empire and Armenia?
Actual Answer: Seljuk Turks
Predicted Answer:
Context : Soon after the Normans began to enter Italy, they entered the Byzantine Empire and then Armenia, fighting against the Pechenegs, the Bulgars, and especially the Seljuk Turks. Norman mercenaries were first encouraged to come to the south by the Lombards to act against the Byzantines, but they soon fought in Byzantine service in Sicily. They were prominent alongside Varangian and Lombard contingents in the Sicilian campaign of George Maniaces in 1038–40. There is debate whether the Normans in Greek service actually were from Norman Italy, and it now seems likely only a few came from there. It is also unknown how many of the "Franks", as the Byzantines called them, were Normans and not other Frenchmen.
Question: Who was the Normans' main enemy in Italy, the Byzantine Empire and Armenia?
Actual Answer: the Pechenegs, the Bulgars, and especially the Seljuk Turks
Predicted Answer:
Context : Soon after the Normans began to enter Italy, they entered the Byzantine Empire and then Armenia, fighting against the Pechenegs, the Bulgars, and especially the Seljuk Turks. Norman mercenaries were first encouraged to come to the south by the Lombards to act against the Byzantines, but they soon fought in Byzantine service in Sicily. They were prominent alongside Varangian and Lombard contingents in the Sicilian campaign of George Maniaces in 1038–40. There is debate whether the Normans in Greek service actually were from Norman Italy, and it now seems likely only a few came from there. It is also unknown how many of the "Franks", as the Byzantines called them, were Normans and not other Frenchmen.
Question: Who was the Normans' main enemy in Italy, the Byzantine Empire and Armenia?
Actual Answer: the Seljuk Turks
Predicted Answer:
Context : One of the first Norman mercenaries to serve as a Byzantine general was Hervé in the 1050s. By then however, there were already Norman mercenaries serving as far away as Trebizond and Georgia. They were based at Malatya and Edessa, under the Byzantine duke of Antioch, Isaac Komnenos. In the 1060s, Robert Crispin led the Normans of Edessa against the Turks. Roussel de Bailleul even tried to carve out an independent state in Asia Minor with support from the local population, but he was stopped by the Byzantine general Alexius Komnenos.
Question: When did Herve serve as a Byzantine general?
Actual Answer: 1050s
Predicted Answer: 1050s by then however there were already norman mercenaries serving as far away as trebizond and georgia they were based at malatya and edessa under the byzantine duke of antioch isaac komnenos in the 1060s