%%capture
!pip install plox
!pip install tabulate
!date
!wget https://raw.githubusercontent.com/mhjabreel/CharCnn_Keras/master/data/ag_news_csv/train.csv
import pandas as pd
df = pd.read_csv("train.csv", header=None)
df.info()
df.head()
df.columns = ["label", "title", "lead"]
label_map = {1: "world", 2: "sport", 3: "business", 4: "sci/tech"}
df["label"] = df["label"].apply(label_map.get)
df.head()
# a new column `text` which contains the lowercased `title` and `lead`
df["text"] = (df.title + " " + df.lead).str.lower()
print(df.text[0])
# print the number of documents for each label
df.label.groupby(df.label).count()
# create a new column with the number of words for each text
# plot the average number of words per label
import re
from plox import Plox
def words(text: str):
return list(filter(len, re.findall("([a-z][a-z'-]*)", str.lower(text))))
df['nwords'] = df.text.apply(lambda text: len(words(text)))
avg_nwords = df.groupby(by='label').nwords.mean()
with Plox() as px:
px.a.bar(x=avg_nwords.index, height=avg_nwords)
px.a.set_ylabel("Average number of words")
px.a.set_xlabel("Text label")
px.show()
from collections import Counter
keywords = ["play", "tax", "blackberry", "israel"]
counts = pd.DataFrame(
data={
keyword:
df.text.apply(
lambda text: Counter(words(text))[keyword]
).groupby(df.label).sum()
for keyword in keywords
},
)
counts['total'] = counts.T.sum(axis=1)
print(counts.to_markdown())
with Plox() as px:
counts.plot(kind='bar', y='tax', ax=px.a)
px.show()
