import nltk
nltk.download("stopwords")
import re
import numpy as np
import pandas as pd
from pprint import pprint
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
import spacy
import pyLDAvis
import pyLDAvis.gensim
import matplotlib.pyplot as plt
%matplotlib inline
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)
import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)
# ready stopwords
from nltk.corpus import stopwords
stop_words = stopwords.words("english")
stop_words.extend(["from", "subject", "re", "edu", "use"])
newsgroups = pd.read_json("https://raw.githubusercontent.com/selva86/datasets/master/newsgroups.json")
print(newsgroups.target_names.unique())
newsgroups.head()
# remove extraneous information
data = newsgroups.content.values.tolist()
data = [re.sub('\S*@\S*\s?', '', sent) for sent in data] # remove emails
data = [re.sub('\s+', ' ', sent) for sent in data] # remove newline
data = [re.sub("\'", "", sent) for sent in data] # remove single quotes
pprint(data[:1])
# tokenize
def sent_to_words(sentences):
for s in sentences:
yield(gensim.utils.simple_preprocess(str(s), deacc=True))
data_words = list(sent_to_words(data))
print(data_words[:1])
# build bigram and trigram models
bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100)
trigram = gensim.models.Phrases(bigram[data_words], threshold=100)
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)
print(trigram_mod[bigram_mod[data_words[0]]])
def remove_stopwords(texts):
return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]
def make_bigrams(texts):
return [bigram_mod[doc] for doc in texts]
def make_trigrams(texts):
return [trigram_mod[bigram_mod[doc]] for doc in texts]
def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
texts_out = []
for sent in texts:
doc = nlp(" ".join(sent))
texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
return texts_out
data_words_nostops = remove_stopwords(data_words) # remove stopwords
data_words_bigrams = make_bigrams(data_words_nostops) # form bigrams
nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner']) # initialize spacey 'en' model
data_lemmatized = lemmatization(data_words_bigrams)
print(data_lemmatized[:1])
id2word = corpora.Dictionary(data_lemmatized) # create dictionary
texts = data_lemmatized # corpus
corpus = [id2word.doc2bow(text) for text in texts]
print(corpus[:1])
# build the lda model
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
id2word=id2word,
num_topics=15,
random_state=100,
update_every=1,
chunksize=100,
passes=10,
alpha='auto',
per_word_topics=True)
pprint(lda_model.print_topics())
doc_lda = lda_model[corpus]
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis
