Merging all Articles

I scheduled this notebook for 3 times a week. Then merged every dataframe into one big dataset called 'combined.csv'. Before merging datasets, each data is cleaned in their individual notebooks. All dates were converted into datetime. Missing descriptions were replaced with 'No Description', nans in the category columns were filled. After merging, the dataframe is also cleaned to drop duplicates, drop excess columns that were created when merged.

import os import pandas as pd import numpy as np from pathlib import Path from os import listdir import glob from datetime import datetime import os.path

os.listdir('/datasets/webscrape/')

os.chdir(os.path.dirname('/datasets/webscrape/')) extension = 'csv' all_filenames = [i for i in glob.glob('*.{}'.format(extension))] for filename in all_filenames: if filename != 'send.csv' and 'new_articles.csv': combined = pd.concat([pd.read_csv(f) for f in all_filenames], ignore_index=True) os.chdir(os.path.dirname('/datasets/webscrape/'))

combined

combined = combined.drop_duplicates(keep='last', subset=['Title'])

combined

combined = combined.drop(columns=['id', 'Unnamed: 0', 'X', 'y', 'prob'])

combined['Description'] = combined['Description'].fillna('No description')

combined = combined.dropna(axis=0, subset=['Date'])

combined['Date'] = combined['Date'].apply(lambda x: datetime.strptime(x,'%Y-%m-%d'))

combined['Category'] = combined['Category'].replace(['Economic', 'Social','Ярилцлага','Аялал','Ulaanbaatar'],['Economics', 'Society','Interview','Travel','Domestic'])

combined['Category'][102:137] = combined['Category'][102:137].fillna('International')

combined['Category'] = combined['Category'].fillna('Finance')

combined = combined.dropna(axis=1)

combined = combined.reset_index()

combined = combined.drop(columns = ['index'])

combined

#combined.to_csv( "/work/merged.csv", index=False, encoding='utf-8-sig')

combined.to_csv('/datasets/webscrape/combined.csv', index=False)

Find Old/New Rows

When I though I had a good amount of articles to train (804 articles) I manually inserted 1's an 0's into a target column of that dataset which shows which articles I would want to send out to people and which ones I wouldn't and named that dataset 'combined_csv'.

The dataset 'combined_csv' will become my old dataset, and then the newly merged dataset would just be called 'combined'. Then I loop through the files and get the recent file which would contain old and new articles. Then I would get the length of the old dataset and slice the newly combined dataset to get a dataframe called 'new_articles'. Then I save the new_article to a csv, concat the dataframe to the old dataset for when the next new articles merge it would become the old dataset.

old_combined = pd.read_csv("/datasets/webscrape/combined_csv")

#old_combined = old_combined.dropna(axis= 1)

#old_combined = old_combined.drop(columns = ['Unnamed: 0.1'])

old_combined

os.chdir(os.path.dirname('/datasets/webscrape/')) extension = 'csv' all_filenames = [i for i in glob.glob('*.{}'.format(extension))] max_file = max(all_filenames, key=os.path.getctime) print(max_file)

new_combined = pd.read_csv(max_file)

new_combined

old_combined

new_articles = new_combined[len(old_combined):]

new_articles = new_articles.reset_index(drop=True)

new_articles

new_articles.to_csv('/datasets/webscrape/new_articles.csv', index=False)

old_combined = pd.concat([old_combined, new_articles])

old_combined.to_csv('/datasets/webscrape/combined_csv.csv')