Day 12

import pandas as pd import numpy as np import matplotlib.pyplot as plt

tweets0 = pd.read_csv('/work/SI_330_Day_12_Text_Processing_II (1)/tweets_labelled_09042020_16072020.csv', sep = ';') tweets1 = pd.read_csv('/work/SI_330_Day_12_Text_Processing_II (1)/tweets_remaining_09042020_16072020.csv', sep=';') tweets1

idint64

created_atobject

2020-04-09 23:59:51+00:00

2020-04-09 23:58:55+00:00

2020-04-09 23:58:52+00:00

2020-04-09 23:58:27+00:00

2020-04-09 23:57:59+00:00

2020-04-09 23:57:33+00:00

2020-04-09 23:57:27+00:00

2020-04-09 23:57:08+00:00

2020-04-09 23:57:01+00:00

2020-04-09 23:56:58+00:00

tweets1.columns = ['id','created_at', 'text'] tweets = pd.concat([tweets0, tweets1]) tweets # There are 928,673 rows and 4 columns

idint64

created_atobject

77522

2020-04-15 01:03:46+00:00

661634

2020-06-25 06:20:06+00:00

413231

2020-06-04 15:41:45+00:00

760262

2020-07-03 19:39:35+00:00

830153

2020-07-09 14:39:14+00:00

27027

2020-04-12 21:52:56+00:00

472959

2020-06-09 05:23:06+00:00

392845

2020-06-02 01:12:29+00:00

313771

2020-05-07 04:58:41+00:00

267894

2020-05-04 15:16:29+00:00

tweets.text.str.startswith('RT').sum()

graph = tweets.text.apply(len).plot(kind='hist') plt.xlabel('Length of tweets') plt.title('Tweet length')

tweets['when'] = pd.to_datetime(tweets['created_at']) tweets.set_index('when').resample('1W').count()

idint64

0 - 93421

created_atint64

0 - 93421

2020-04-12T00:00:00.000000

32193

2020-04-19T00:00:00.000000

90415

2020-04-26T00:00:00.000000

84846

2020-05-03T00:00:00.000000

52126

2020-05-10T00:00:00.000000

71721

2020-05-17T00:00:00.000000

2020-05-24T00:00:00.000000

2020-05-31T00:00:00.000000

34668

2020-06-07T00:00:00.000000

75730

2020-06-14T00:00:00.000000

73682

tweets['text'] = tweets.text.str.findall('(@\w+)').apply(len) tweets.text.plot.hist()

tweets['hashtag_count'] = tweets['text'].str.count("#\w+") tweets.sort_values('hashtag_count', ascending=False)

idint64

created_atobject

203047

206308

2020-04-26 14:41:37+00:00

309717

314701

2020-05-07 02:17:26+00:00

256079

260228

2020-05-03 10:02:18+00:00

303438

308313

2020-05-07 15:57:08+00:00

481968

489793

2020-06-10 04:47:36+00:00

531587

540173

2020-06-16 16:30:27+00:00

305322

310228

2020-05-07 13:45:04+00:00

216933

220446

2020-04-27 07:49:07+00:00

822618

835920

2020-07-09 05:03:29+00:00

798247

811142

2020-07-08 17:29:58+00:00

tweets['hash_count'] = tweets['text'].str.findall("#\w+") tweets.explode('hash_count')['hash_count'].str.lower().value_counts().head(10)

tweets['stock_symbols'] = tweets['text'].str.findall('^\$[a-zA-Z]+\.*[a-zA-Z]+') tweets.explode('stock_symbols')['stock_symbols'].str.lower().value_counts().sort_values(ascending=False).head(10)

# insert your code here