News Reader

I have scrapped the headlines of the news related to technology and science from various magazine, newspapers and websites.

to collect dataset of the recent technology news from different resources, and the applying some NLP techniques in order to analyze to most common trend of technology.

then we would do "Topic Modeling". Topic Modeling is an unsupervised learning approach to clustering documents, to discover topics based on their contents. It is very similar to how K-Means algorithm and Expectation-Maximization work.

Libraries

import os import urllib from bs4 import BeautifulSoup import subprocess import os from time import sleep import re from datetime import date import pandas as pd import requests from newspaper import Article

Run to view results

Data scraping

The data was acquired from many websites such as: inshort, linux magazine, BBC technology, google news AI canada, yahoo tech and google news computing canada.

The data is scrapped as html file, parsed using BeautifulSoup, and then the relevant information is extracted and stored in csv file.

Depending on the accessibility of each website, sometimes the urllib is used and sometimes requests,

i.e the request using urllib is forbidden in some website.

inshort

ishort_data=[] ishort_html=urllib.request.urlopen('https://inshorts.com/en/read/technology')

Run to view results

ishort_soup=BeautifulSoup(ishort_html.read()) ishort_news=ishort_soup.find_all('div',attrs={'itemprop':'articleBody'}) ishort_title=ishort_soup.find_all('span',attrs={'itemprop':'headline'}) for item in zip(ishort_title,ishort_news): title=item[0].get_text() text=item[1].get_text() ishort_data.append({'text':text, 'auhtor':'unkown', 'newspaper':'inshort', 'country':'india', 'date': str(date.today()), })

Run to view results

df = pd.DataFrame(ishort_data) df

Run to view results

Linux magazine

linux_data=[] linux_html=urllib.request.urlopen('https://www.linux-magazine.com/Online/News')

Run to view results

linux_soup=BeautifulSoup(linux_html.read(),'html.parser') linux_news=linux_soup.find_all('p') for p in linux_news: linux_data.append({'text':p.get_text().strip(), 'auhtor':'unkown', 'newspaper':'linux', 'country':'USA', 'date': str(date.today())})

Run to view results

linux_df = pd.DataFrame(linux_data) linux_df

Run to view results

Google AI news

data_google_ai = [] # google news AI canada #h4 gPFEn data_google_ai = [] html_google = requests.get('https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGRqTVhZU0FtVnVHZ0pEUVNnQVAB/sections/CAQiR0NCQVNMd29JTDIwdk1EZGpNWFlTQldWdUxVZENHZ0pEUVNJTkNBUWFDUW9ITDIwdk1HMXJlaW9KRWdjdmJTOHdiV3Q2S0FBKi4IACoqCAoiJENCQVNGUW9JTDIwdk1EZGpNWFlTQldWdUxVZENHZ0pEUVNnQVABUAE?hl=en-CA&gl=CA&ceid=CA%3Aen') news_google = BeautifulSoup(html_google.text,"html.parser").find_all('a', {'class':'WwrzSb'})

Run to view results

news_google[0].get('href')

Run to view results

for p in news_google: try: headline_url='https://news.google.com'+p.get('href')[1:] article=Article(headline_url) article.download() article.parse() try: d =str(article.publish_date.date()) except: d='' try: auth = ' '.join(article.authors) except: auth='' data_google_ai.append({ 'text':str(article.title.strip()), 'auhtor':'unkown','newspaper':'google_news','country':'canada','date': d,'auhtor':auth}) except: pass

Run to view results

google_ai_df = pd.DataFrame(data_google_ai) google_ai_df

Run to view results

BBC news

bbc_data=[] bbc_html=requests.get('https://www.bbc.com/news/technology') bbc_soup=BeautifulSoup(bbc_html.text,'html.parser')

Run to view results

bbc_news=bbc_soup.find_all('p',{'class':'lx-stream-related-story--summary qa-story-summary'}) for p in bbc_news: bbc_data.append({'text':p.get_text(),'author':'unkown','newspaper':'BBC news','country':'UK','date':str(date.today())})

Run to view results

bbc_df=pd.DataFrame(bbc_data)

Run to view results

bbc_df

Run to view results

Yahoo tech news

you need to create an account in yahoo news to get access to the data

email='mustaphaamine896@gmail.com' password='anoldpassword@1234'

Run to view results

yahoo_data=[] cred={'username':email,'password':password} login_url = 'https://login.yahoo.com/?.intl=in&src=noSrc&intl=en&specId=yidReg&done=https%3A%2F%2Fwww.yahoo.com&nr=1' s=requests.Session() respond=s.post(login_url,cred) yahoo_html=s.get('https://uk.finance.yahoo.com/')

Run to view results

yahoo_soup=BeautifulSoup(yahoo_html.text,'html.parser') yahoo_news=yahoo_soup.find_all('p',{'class':'Fz(14px) Lh(19px) Fz(13px)--sm1024 Lh(17px)--sm1024 LineClamp(2,38px) LineClamp(2,34px)--sm1024 M(0)'}) news_au=yahoo_soup.find_all('div',{'class':'C(#959595) Fz(11px) D(ib) Mb(6px)'}) for p in zip(yahoo_news[:],news_au[:]): yahoo_data.append({'text': p[0].get_text().strip(), 'author':p[1].get_text().strip(), 'newspaper':'yahoo', 'country':'UK' ,'date':str(date.today())} )

Run to view results

yahoo_df=pd.DataFrame(yahoo_data) yahoo_df

Run to view results

Google Tech news

data_tech_google = [] html_google = requests.get('https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGRqTVhZU0FtVnVHZ0pEUVNnQVAB?hl=en-CA&gl=CA&ceid=CA%3Aen')

Run to view results

news_google = BeautifulSoup(html_google.text,"html.parser").find_all('a', {'class':'WwrzSb'}) news_autor = BeautifulSoup(html_google.text,"html.parser").find_all('div', {'class':'oovtQ'}) #div oovtQ for p in zip(news_autor[:],news_google[:]): try: headline_url = 'https://news.google.com'+p[1].get('href')[1:] article = Article(headline_url) article.download() article.parse() try: d =str(article.publish_date.date()) except: d='' try: auth = ' '.join(article.authors) except: auth='' #print(article.title) data_tech_google.append({ 'text':str(article.title.strip()), 'auhtor':p[0].get_text(), 'newspaper':'google_news', 'country':'canada', 'date': d, 'auhtor':auth}) except: pass

Run to view results

google_tech_news=pd.DataFrame(data_tech_google) google_tech_news

Run to view results

Merging all the news data

#Merging all Scrapped data into one dataset. full_data=pd.concat([pd.DataFrame(df),pd.DataFrame(linux_df),pd.DataFrame(google_ai_df),pd.DataFrame(bbc_df),pd.DataFrame(yahoo_df),pd.DataFrame(google_tech_news)])

Run to view results

full_data

Run to view results

#Saving the results as csv file path = "./dataset" isExist = os.path.exists(path) if not isExist: os.makedirs(path) print("The dataset directory is created!") full_data.to_csv('dataset/news_data.csv')

Run to view results