# Importing modules
# -- YOUR CODE HERE --
import pandas as pd
# Read datasets/papers.csv into papers
papers = pd.read_csv('datasets/papers.csv')
# Print out the first rows of papers
# -- YOUR CODE HERE --
print(papers.head())
id year title event_type \
0 1 1987 Self-Organization of Associative Database and ... NaN
1 10 1987 A Mean Field Theory of Layer IV of Visual Cort... NaN
2 100 1988 Storing Covariance by the Associative Long-Ter... NaN
3 1000 1994 Bayesian Query Construction for Neural Network... NaN
4 1001 1994 Neural Network Ensembles, Cross Validation, an... NaN
pdf_name abstract \
0 1-self-organization-of-associative-database-an... Abstract Missing
1 10-a-mean-field-theory-of-layer-iv-of-visual-c... Abstract Missing
2 100-storing-covariance-by-the-associative-long... Abstract Missing
3 1000-bayesian-query-construction-for-neural-ne... Abstract Missing
4 1001-neural-network-ensembles-cross-validation... Abstract Missing
paper_text
0 767\n\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...
1 683\n\nA MEAN FIELD THEORY OF LAYER IV OF VISU...
2 394\n\nSTORING COVARIANCE BY THE ASSOCIATIVE\n...
3 Bayesian Query Construction for Neural\nNetwor...
4 Neural Network Ensembles, Cross\nValidation, a...
# Remove the columns
# -- YOUR CODE HERE --
papers.drop(columns=['id','event_type','pdf_name' ], inplace=True)
# Print out the first rows of papers
# -- YOUR CODE HERE --
print(papers.head())
year title abstract \
0 1987 Self-Organization of Associative Database and ... Abstract Missing
1 1987 A Mean Field Theory of Layer IV of Visual Cort... Abstract Missing
2 1988 Storing Covariance by the Associative Long-Ter... Abstract Missing
3 1994 Bayesian Query Construction for Neural Network... Abstract Missing
4 1994 Neural Network Ensembles, Cross Validation, an... Abstract Missing
paper_text
0 767\n\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...
1 683\n\nA MEAN FIELD THEORY OF LAYER IV OF VISU...
2 394\n\nSTORING COVARIANCE BY THE ASSOCIATIVE\n...
3 Bayesian Query Construction for Neural\nNetwor...
4 Neural Network Ensembles, Cross\nValidation, a...
# Group the papers by year
groups = papers.groupby('year')
# Determine the size of each group
counts = groups.size()
# Visualise the counts as a bar plot
import matplotlib.pyplot as plt
%matplotlib inline
# -- YOUR CODE HERE --
counts.plot()
plt.title("Machine Learning Publications since 1987")
plt.xlabel("Year");
# Load the regular expression library
# -- YOUR CODE HERE --
import re
# Print the titles of the first rows
print(papers['title'].head())
# Remove punctuation
papers['title_processed'] = papers['title'].map(lambda x: re.sub('[,\.!?]', '', x))
# Convert the titles to lowercase
papers['title_processed'] = papers['title_processed'].map(lambda x: x.lower())
# Print the processed titles of the first rows
# -- YOUR CODE HERE --
print(papers['title_processed'])
0 Self-Organization of Associative Database and ...
1 A Mean Field Theory of Layer IV of Visual Cort...
2 Storing Covariance by the Associative Long-Ter...
3 Bayesian Query Construction for Neural Network...
4 Neural Network Ensembles, Cross Validation, an...
Name: title, dtype: object
0 self-organization of associative database and ...
1 a mean field theory of layer iv of visual cort...
2 storing covariance by the associative long-ter...
3 bayesian query construction for neural network...
4 neural network ensembles cross validation and ...
5 using a neural net to instantiate a deformable...
6 plasticity-mediated competitive learning
7 iceg morphology classification using an analog...
8 real-time control of a tokamak plasma using ne...
9 pulsestream synapses with non-volatile analogu...
10 learning to play the game of chess
11 multidimensional scaling and data clustering
12 an experimental comparison of recurrent neural...
13 training multilayer perceptrons with the exten...
14 interference in learning internal models of in...
15 active learning with statistical models
16 a rapid graph-based method for arbitrary trans...
17 ocular dominance and patterned lateral connect...
18 associative decorrelation dynamics: a theory o...
19 a connectionist technique for accelerated text...
20 connectionist speaker normalization with gener...
21 a critical comparison of models for orientatio...
22 generalization in reinforcement learning: safe...
23 a mixture model system for medical and machine...
24 an application of the principle of maximum inf...
25 a computational model of prefrontal cortex fun...
26 the gamma mlp for speech phoneme recognition
27 a multiscale attentional framework for relaxat...
28 correlated neuronal response: time scales and ...
29 onset-based sound segmentation
...
7211 combining estimators using non-constant weight...
7212 a model of the hippocampus combining self-orga...
7213 catastrophic interference in human motor learning
7214 model of a biological neuron as a temporal neu...
7215 dynamic cell structures
7216 on-line learning of dichotomies
7217 new algorithms for 2d and 3d point matching: p...
7218 pattern playback in the 90s
7219 learning in large linear perceptrons and why t...
7220 spreading activation over distributed microfea...
7221 interior point implementations of alternating ...
7222 reinforcement learning with soft state aggrega...
7223 coarse-to-fine image search using neural networks
7224 on the computational utility of consciousness
7225 a convolutional neural network hand tracker
7226 efficient methods for dealing with missing dat...
7227 grammar learning by a self-organizing network
7228 recurrent networks: second order properties an...
7229 comparing the prediction accuracy of artificia...
7230 convergence properties of the k-means algorithms
7231 analog implementation of shunting neural networks
7232 learning with product units
7233 stochastic dynamics of three-state neural netw...
7234 grouping components of three-dimensional movin...
7235 visual speech recognition with stochastic netw...
7236 single transistor learning synapses
7237 bias variance and the combination of least squ...
7238 a real time clustering cmos neural engine
7239 learning direction in global motion: two class...
7240 correlation and interpolation networks for rea...
Name: title_processed, Length: 7241, dtype: object
# Import the wordcloud library
# -- YOUR CODE HERE --
import wordcloud
# Join the different processed titles together.
long_string = " ".join(papers.title_processed)
# Create a WordCloud object
wordcloud = wordcloud.WordCloud()
# Generate a word cloud
# -- YOUR CODE HERE --
wordcloud.generate(long_string)
# Visualize the word cloud
wordcloud.to_image()
# Load the library with the CountVectorizer method
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
# Helper function
def plot_10_most_common_words(count_data, count_vectorizer):
import matplotlib.pyplot as plt
words = count_vectorizer.get_feature_names()
total_counts = np.zeros(len(words))
for t in count_data:
total_counts+=t.toarray()[0]
count_dict = (zip(words, total_counts))
count_dict = sorted(count_dict, key=lambda x:x[1], reverse=True)[0:10]
words = [w[0] for w in count_dict]
counts = [w[1] for w in count_dict]
x_pos = np.arange(len(words))
plt.bar(x_pos, counts,align='center')
plt.xticks(x_pos, words, rotation=90)
plt.xlabel('words')
plt.ylabel('counts')
plt.title('10 most common words')
plt.show()
# Initialise the count vectorizer with the English stop words
count_vectorizer = CountVectorizer(stop_words='english')
# Fit and transform the processed titles
count_data = count_vectorizer.fit_transform(papers['title_processed'])
# Visualise the 10 most common words
# -- YOUR CODE HERE --
plot_10_most_common_words(count_data, count_vectorizer)
import warnings
warnings.simplefilter("ignore", DeprecationWarning)
# Load the LDA model from sk-learn
from sklearn.decomposition import LatentDirichletAllocation as LDA
# Helper function
def print_topics(model, count_vectorizer, n_top_words):
words = count_vectorizer.get_feature_names()
for topic_idx, topic in enumerate(model.components_):
print("\nTopic #%d:" % topic_idx)
print(" ".join([words[i]
for i in topic.argsort()[:-n_top_words - 1:-1]]))
# Tweak the two parameters below (use int values below 15)
number_topics = 5
number_words = 5
# Create and fit the LDA model
lda = LDA(n_components=number_topics)
lda.fit(count_data)
# Print the topics found by the LDA model
print("Topics found via LDA:")
print_topics(lda, count_vectorizer, number_words)
Topics found via LDA:
Topic #0:
learning data models markov clustering
Topic #1:
learning models multi classification non
Topic #2:
learning sparse stochastic optimal algorithms
Topic #3:
networks neural learning inference optimization
Topic #4:
analysis estimation online algorithm feature
# The historical data indicates that:
more_papers_published_in_2018 = True