def collectSportsNews():
URL = "https://indianexpress.com/section/sports/"
r = requests.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
news_list = []
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "SPORTS"))
for i in range(2,100):
r = requests.get(URL+"/page/"+str(i))
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "SPORTS"))
connection.commit()
print(news_list)
def collectEntertainmentData():
URL = "https://indianexpress.com/section/entertainment/"
r = requests.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
news_list = []
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "ENTERTAINMENT"))
for i in range(2,100):
r = requests.get(URL+"/page/"+str(i))
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "ENTERTAINMENT"))
connection.commit()
print(news_list)
def collectTechData():
URL = "https://indianexpress.com/section/technology/"
r = requests.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
# selected = soup.find('ul', attrs = {'class':'article-list'})
# newss = selected.findAll('li')
# for news in newss:
# c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.findAll('a')[1].text, "TECH"))
for i in range(100,106):
r = requests.get(URL+"/page/"+str(i))
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('ul', attrs = {'class':'article-list'})
newss = selected.findAll('li')
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.findAll('a')[1].text, "TECH"))
connection.commit()
def collectBusinessData():
URL = "https://indianexpress.com/section/business/"
r = requests.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
news_list = []
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "BUSINESS"))
for i in range(2,100):
r = requests.get(URL+"/page/"+str(i))
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "BUSINESS"))
connection.commit()
print(news_list)
def collectEducationData():
URL = "https://indianexpress.com/section/education/"
r = requests.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
news_list = []
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "EDUCATION"))
for i in range(2,100):
r = requests.get(URL+"/page/"+str(i))
soup = BeautifulSoup(r.content, 'html5lib')
selected = soup.find('div', attrs = {'class':'nation'})
newss = selected.findAll('div', attrs={'class': 'articles'})
for news in newss:
c.execute("INSERT INTO NEWS(news_article , category ) VALUES (?,?)",(news.find('p').text, "EDUCATION"))
connection.commit()
print(news_list)
def getAllInfoData():
c.execute("SELECT Category , COUNT(*) FROM NEWS Group by Category")
print(c.fetchall())
!pip install bs4
Collecting bs4
Downloading bs4-0.0.1.tar.gz (1.1 kB)
Collecting beautifulsoup4
Downloading beautifulsoup4-4.10.0-py3-none-any.whl (97 kB)
|████████████████████████████████| 97 kB 9.8 MB/s
Collecting soupsieve>1.2
Downloading soupsieve-2.3.1-py3-none-any.whl (37 kB)
Building wheels for collected packages: bs4
Building wheel for bs4 (setup.py) ... done
Created wheel for bs4: filename=bs4-0.0.1-py3-none-any.whl size=1272 sha256=21283b0aa2c07c75db181742a81e6192ef8b9c342d6a77ed6de0faff63b04356
Stored in directory: /root/.cache/pip/wheels/0a/9e/ba/20e5bbc1afef3a491f0b3bb74d508f99403aabe76eda2167ca
Successfully built bs4
Installing collected packages: soupsieve, beautifulsoup4, bs4
Successfully installed beautifulsoup4-4.10.0 bs4-0.0.1 soupsieve-2.3.1
WARNING: You are using pip version 20.1.1; however, version 21.3.1 is available.
You should consider upgrading via the '/root/venv/bin/python -m pip install --upgrade pip' command.
import sqlite3
import requests
from bs4 import BeautifulSoup
import os
# if db not exist then ask and scrape
if not (os.path.isfile("News.db")):
print("#"*50)
choice = input("Can I scrape the data from net? Y-n: ")
if (choice.lower() == "y" or choice.lower() ==" yes"):
connection = sqlite3.connect("News.db")
c = connection.cursor()
c.execute("""CREATE TABLE NEWS(news_article text, category text)""")
collectSportsNews()
collectEntertainmentData()
collectTechData()
collectEducationData()
collectBusinessData()
getAllInfoData()
else:
pass
else:
connection = sqlite3.connect("News.db")
c = connection.cursor()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pickle
sql_query = pd.read_sql_query ('''
SELECT
*
FROM NEWS
''', connection)
df = pd.DataFrame(sql_query, columns = ['news_article', 'category'])
# print (df)
# getAllInfoData()
df.head()
news_articleobject
categoryobject
0
Former Bengal captain Laxmi Ratan Shukla, who is also the current Under-23 coach of the state team,…
SPORTS
1
"Day 40 - Finished. Preet has just made history becoming the first woman of colour to complete a so…
SPORTS
2
Jan-Lennard Struff withstood 34 aces in a 7-6 (7), 4-6, 7-5 win over John Isner to give Germany the…
SPORTS
3
On a pitch on which Bangladesh and New Zealand collectively amassed 786 runs in their first innings…
SPORTS
4
The matter was, however, resolved after talks between players' representatives, the England and Wal…
SPORTS
This chart is empty
Chart was probably not set up properly in the notebook
df.category.value_counts()
max(df["news_article"])
min(df["news_article"])
df.isnull().sum()
from sklearn.preprocessing import LabelEncoder
lc = LabelEncoder()
df["categoryId"] = lc.fit_transform(df["category"])
df.sample(10)
news_articleobject
Rajasthan BSTC 2021 exam will be held on August 31, 2021. The admit card will be available for download at predeled.com. The exam is conducted at the state level and around 350 colleges allocate seats based on Rajasthan BSTC scores. 10%
Sahal Abdul Samad (27th), Alvaro Vazquez (47th) and Jorge Pereyra Diaz (51st) got on the scoresheet for Kerala Blasters who moved to fifth spot in the standings with nine points from six games.10%
8 others80%
categoryobject
TECH30%
EDUCATION20%
3 others50%
10412
Rajasthan BSTC 2021 exam will be held on August 31, 2021. The admit card will be available for down…
EDUCATION
428
Sahal Abdul Samad (27th), Alvaro Vazquez (47th) and Jorge Pereyra Diaz (51st) got on the scoresheet…
SPORTS
2953
Spider-Man: No Way Home was supposed to be an Endgame-level crossover event for fans of the superhe…
ENTERTAINMENT
1321
After a clean sweep against New Zealand in the T20 series, India will be locking horns against the …
SPORTS
11480
Candidates may now submit the form by June 10, without incurring a late fee at the official website…
EDUCATION
7932
Share Market, Stock Market Highlights: The S&P BSE Sensex ended at a fresh lifetime high of 60,048.…
BUSINESS
2802
While a local parents-teachers' body raised objections to it, several netizens shared the copy of t…
ENTERTAINMENT
6329
Tech launches of the week: Lenovo Yoga Tab 11, Nokia C30, and more
TECH
12312
Facebook rolls out new messaging, business tools for brands
TECH
5549
Twitter expands safety policy, disallows sharing of private media without consent
TECH
pickle.dump(lc, open('helpers/LabelEncoder.pickel', 'wb'))
categoryDF = df[["category", "categoryId"]].drop_duplicates().sort_values(by="categoryId")
categoryDF
categoryobject
categoryIdint64
6930
BUSINESS
0
9405
EDUCATION
1
2475
ENTERTAINMENT
2
0
SPORTS
3
4950
TECH
4
df["category"].value_counts().plot(kind="pie", shadow=True, autopct='%1.1f%%')
df["category"].value_counts().plot(kind="bar")
!pip install wordcloud
Collecting wordcloud
Downloading wordcloud-1.8.1-cp37-cp37m-manylinux1_x86_64.whl (366 kB)
|████████████████████████████████| 366 kB 20.6 MB/s
Requirement already satisfied: matplotlib in /shared-libs/python3.7/py/lib/python3.7/site-packages (from wordcloud) (3.5.1)
Requirement already satisfied: numpy>=1.6.1 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from wordcloud) (1.19.5)
Requirement already satisfied: pillow in /shared-libs/python3.7/py/lib/python3.7/site-packages (from wordcloud) (8.4.0)
Requirement already satisfied: kiwisolver>=1.0.1 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from matplotlib->wordcloud) (1.3.2)
Requirement already satisfied: cycler>=0.10 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from matplotlib->wordcloud) (0.11.0)
Requirement already satisfied: fonttools>=4.22.0 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from matplotlib->wordcloud) (4.28.3)
Requirement already satisfied: packaging>=20.0 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from matplotlib->wordcloud) (21.3)
Requirement already satisfied: python-dateutil>=2.7 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from matplotlib->wordcloud) (2.8.2)
Requirement already satisfied: pyparsing>=2.2.1 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from matplotlib->wordcloud) (3.0.6)
Requirement already satisfied: six>=1.5 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib->wordcloud) (1.16.0)
Installing collected packages: wordcloud
Successfully installed wordcloud-1.8.1
WARNING: You are using pip version 20.1.1; however, version 21.3.1 is available.
You should consider upgrading via the '/root/venv/bin/python -m pip install --upgrade pip' command.
import nltk
from wordcloud import WordCloud, STOPWORDS
from nltk.corpus import stopwords
# Create stopword list:
stopwords = set(STOPWORDS)
stopwords.update(["br", "href"])
textt = " ".join(review for review in df["news_article"])
wordcloud = WordCloud(stopwords=stopwords).generate(textt)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.savefig('images/wordcloud11.png')
plt.show()
"' '".join(set(" ".join(review for review in df["news_article"])))
# REMOVE SPECIAL CHARs
import re
def filterNonAscii(text):
split = re.split("\W+", text)
return " ".join(split)
df["filter1"]= df["news_article"].apply(lambda x: filterNonAscii(x.lower()))
"' '".join(set(" ".join(review for review in df["filter1"])))
nltk.download('stopwords')
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data] Unzipping corpora/stopwords.zip.
def remove_stopwords(text):
stopwords = set(nltk.corpus.stopwords.words('english'))
stopwords.update(["br", "href"])
text=[word for word in text.split(" ") if word not in stopwords]
return " ".join(text)
df["filter1"]= df["filter1"].apply(lambda x: remove_stopwords(x))
print(df["news_article"][4])
print("-"*170)
print(df["filter1"][4])
The matter was, however, resolved after talks between players' representatives, the England and Wales Cricket Board (ECB), Cricket Australia and the Australian government.
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
matter however resolved talks players representatives england wales cricket board ecb cricket australia australian government
from nltk.stem import WordNetLemmatizer
def lemmitize(text):
wordnet = WordNetLemmatizer()
return " ".join([wordnet.lemmatize(word) for word in text.split(" ")])
nltk.download('wordnet')
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data] Unzipping corpora/wordnet.zip.
df["filter2"]= df["filter1"].apply(lambda x: lemmitize(x))
print(df["news_article"][4])
print("-"*170)
print(df["filter2"][4])
The matter was, however, resolved after talks between players' representatives, the England and Wales Cricket Board (ECB), Cricket Australia and the Australian government.
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------
matter however resolved talk player representative england wale cricket board ecb cricket australia australian government
sportsDF = df[df["category"]=="SPORTS"]
teachDF = df[df["category"]=="TECH"]
entertainmentDF = df[df["category"]=="ENTERTAINMENT"]
businessDF = df[df["category"]=="BUSINESS"]
educationDF = df[df["category"]=="EDUCATION"]
educationDF.sample(3)
news_articleobject
categoryobject
11742
Hemant Jindal secured all India rank (AIR) 8 in GATE 2021 (electrical). He shares his preparation s…
EDUCATION
10879
WBBSE president Kalyanmoy Ganguly announced that the 100 per cent pass percentage is an all-time re…
EDUCATION
11870
The state government has allotted a budget of Rs 100 crore to the school education department for t…
EDUCATION
textt = " ".join(review for review in sportsDF["filter2"])
wordcloud = WordCloud(stopwords=stopwords).generate(textt)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.savefig('images/sports.png')
plt.show()
textt = " ".join(review for review in educationDF["filter2"])
wordcloud = WordCloud(stopwords=stopwords).generate(textt)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.savefig('images/edu.png')
plt.show()
textt = " ".join(review for review in businessDF["filter2"])
wordcloud = WordCloud(stopwords=stopwords).generate(textt)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.savefig('images/business.png')
plt.show()
textt = " ".join(review for review in teachDF["filter2"])
wordcloud = WordCloud(stopwords=stopwords).generate(textt)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.savefig('images/tech.png')
plt.show()
textt = " ".join(review for review in entertainmentDF["filter2"])
wordcloud = WordCloud(stopwords=stopwords).generate(textt)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.savefig('images/entertainment.png')
plt.show()
from sklearn.feature_extraction.text import CountVectorizer
x = np.array(df["filter2"].values)
y = np.array(df.categoryId.values)
cv = CountVectorizer(max_features = 6000)
x = cv.fit_transform(df.filter2).toarray()
print("X.shape = ",x.shape)
print("y.shape = ",y.shape)
X.shape = (12400, 6000)
y.shape = (12400,)
import pickle
pickle.dump(cv, open('helpers/countVectorizer.pickel', 'wb'))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.3, random_state = 10, shuffle = True)
print(len(x_train))
print(len(x_test))
8680
3720
y_train_sampling = np.unique(y_train, return_counts=True)[1]/len(y_train)*100
y_sampling = np.unique(y, return_counts=True)[1]/len(y)*100
y_train_sampling
y_sampling
np.sum(np.square(y_train_sampling-y_sampling))
# from sklearn.model_selection import train_test_split
# x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.3, random_state = 0, shuffle = True)
# print(len(x_train))
# print(len(x_test))
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.3, random_state=42)
for train_index, test_index in split.split(df, df["categoryId"]):
x_train = x[train_index]
y_train = y[train_index]
x_test = x[test_index]
y_test = y[test_index]
y_train_strat_sampling = np.unique(y_train, return_counts=True)[1]/len(y_train)*100
y_train_strat_sampling
np.sum(np.square(y_train_strat_sampling-y_sampling))
from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import recall_score, precision_score, f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
import seaborn as sns
from sklearn.naive_bayes import MultinomialNB
def trainAModel(mdl):
if(mdl == "logistic"):
mdl = LogisticRegression()
elif mdl == "SVC":
mdl = SVC()
elif mdl == "decissionTree":
mdl = DecisionTreeClassifier()
elif mdl == "randomForest":
mdl = RandomForestClassifier(n_estimators=100 ,criterion='entropy' , random_state=0)
elif mdl == "knearestN":
mdl = KNeighborsClassifier(n_neighbors=10 , metric= 'minkowski' , p = 4)
else:
mdl = MultinomialNB(alpha=1.0,fit_prior=True)
oneVsRest = OneVsRestClassifier(mdl)
oneVsRest.fit(x_train, y_train)
y_pred = oneVsRest.predict(x_test)
accuracy = round(accuracy_score(y_test, y_pred) * 100, 2)
print("Accuracy: "+ str(accuracy))
print("Precission Score: "+ str(classification_report(y_true=y_test, y_pred=y_pred)))
# print("Recall Score: "+ str(recall_score(y_test, y_pred, pos_label='positive' ,average='micro')))
# print("F1 Score: "+ str(f1_score(y_test, y_pred, pos_label='positive' ,average='micro')))
print("ConfusionMatrix: ")
sns.heatmap(confusion_matrix(y_test, y_pred), annot=True)
print()
return oneVsRest
%%time
LogisticMdl = trainAModel("logistic")
Accuracy: 93.44
Precission Score: precision recall f1-score support
0 0.93 0.91 0.92 743
1 0.97 0.93 0.95 742
2 0.96 0.94 0.95 743
3 0.94 0.92 0.93 742
4 0.88 0.97 0.92 750
accuracy 0.93 3720
macro avg 0.94 0.93 0.93 3720
weighted avg 0.94 0.93 0.93 3720
ConfusionMatrix:
CPU times: user 19.6 s, sys: 2.95 s, total: 22.6 s
Wall time: 15.4 s
import pickle
pickle.dump(LogisticMdl, open('models/logisticMdl.sav', 'wb'))
%%time
dTreeMdl = trainAModel("decissionTree")
Accuracy: 82.02
Precission Score: precision recall f1-score support
0 0.95 0.71 0.81 743
1 0.96 0.85 0.90 742
2 0.94 0.80 0.87 743
3 0.91 0.78 0.84 742
4 0.58 0.96 0.73 750
accuracy 0.82 3720
macro avg 0.87 0.82 0.83 3720
weighted avg 0.87 0.82 0.83 3720
ConfusionMatrix:
CPU times: user 4min 57s, sys: 688 ms, total: 4min 58s
Wall time: 5min 1s
pickle.dump(dTreeMdl, open('models/decissionTree.sav', 'wb'))
%%time
naiveMdl = trainAModel("naive")
Accuracy: 93.82
Precission Score: precision recall f1-score support
0 0.91 0.92 0.91 743
1 0.95 0.95 0.95 742
2 0.94 0.97 0.96 743
3 0.94 0.95 0.94 742
4 0.94 0.90 0.92 750
accuracy 0.94 3720
macro avg 0.94 0.94 0.94 3720
weighted avg 0.94 0.94 0.94 3720
ConfusionMatrix:
CPU times: user 2.37 s, sys: 424 ms, total: 2.79 s
Wall time: 2.45 s
pickle.dump(naiveMdl, open('models/naiveMdl.sav', 'wb'))
%%time
svcMdl = trainAModel("SVC")
Accuracy: 94.09
Precission Score: precision recall f1-score support
0 0.93 0.94 0.93 743
1 0.97 0.93 0.95 742
2 0.96 0.94 0.95 743
3 0.95 0.92 0.94 742
4 0.90 0.97 0.93 750
accuracy 0.94 3720
macro avg 0.94 0.94 0.94 3720
weighted avg 0.94 0.94 0.94 3720
ConfusionMatrix:
CPU times: user 12min 32s, sys: 3.99 s, total: 12min 36s
Wall time: 12min 40s
pickle.dump(svcMdl, open('models/svcMdl.sav', 'wb'))
mdl = naiveMdl
def PredictNewsType(news):
if type(news) == str:
news = [news]
toBePredicted = pd.DataFrame(news)
toBePredicted = toBePredicted[0]
toBePredicted = toBePredicted.apply(lambda x: filterNonAscii(x.lower()))
toBePredicted = toBePredicted.apply(lambda x: remove_stopwords(x))
toBePredicted = toBePredicted.apply(lambda x: lemmitize(x))
outs = mdl.predict(cv.transform(toBePredicted).toarray())
outs_prob = mdl.predict_proba(cv.transform(toBePredicted).toarray())
ans= []
for i, out in enumerate(outs):
ans.append((lc.inverse_transform([out])[0], np.round(max(outs_prob[i])*100,2)))
return ans
PredictNewsType(["Before going to Spain, Loh had trained in Dubai with Tokyo Olympics champion Viktor Axelsen and Sen and said the young Indian is an extremely talented player.",
"The second Test saw quite a few heated exchanges between the players and with a fit-again Kohli back for the all-important game, Elgar feels more verbal blows will be traded at the Newlands.",
"China’s Chang’e 5 lunar probe finds first on-site evidence of water on moon’s surface",
"Candidates can check the detailed schedule on the official website of MCC- mcc.nic.in. The registration process for the counselling will begin on January 12.",
"In its pre-Budget memorandum, the Hotel Association of India (HAI) said policy interventions are imperative for the sector's survival and its early and quick rebound to normalcy.",
"Oh Young Soo was nominated alongside Billy Crudup and Mark Duplass in “The Morning Show,” Kieran Culkin in “Succession,” and Brett Goldstein in “Ted Lasso.”",
])
PredictNewsType("Classes will be online since june")
lc.inverse_transform([2])[0]