import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import nltk # you might need to do pip install nltk once only
nltk.download('book') # you only need to do this once
[nltk_data] Downloading collection 'book'
[nltk_data] |
[nltk_data] | Downloading package abc to /root/nltk_data...
[nltk_data] | Unzipping corpora/abc.zip.
[nltk_data] | Downloading package brown to /root/nltk_data...
[nltk_data] | Unzipping corpora/brown.zip.
[nltk_data] | Downloading package chat80 to /root/nltk_data...
[nltk_data] | Unzipping corpora/chat80.zip.
[nltk_data] | Downloading package cmudict to /root/nltk_data...
[nltk_data] | Unzipping corpora/cmudict.zip.
[nltk_data] | Downloading package conll2000 to /root/nltk_data...
[nltk_data] | Unzipping corpora/conll2000.zip.
[nltk_data] | Downloading package conll2002 to /root/nltk_data...
[nltk_data] | Unzipping corpora/conll2002.zip.
[nltk_data] | Downloading package dependency_treebank to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping corpora/dependency_treebank.zip.
[nltk_data] | Downloading package genesis to /root/nltk_data...
[nltk_data] | Unzipping corpora/genesis.zip.
[nltk_data] | Downloading package gutenberg to /root/nltk_data...
[nltk_data] | Unzipping corpora/gutenberg.zip.
[nltk_data] | Downloading package ieer to /root/nltk_data...
[nltk_data] | Unzipping corpora/ieer.zip.
[nltk_data] | Downloading package inaugural to /root/nltk_data...
[nltk_data] | Unzipping corpora/inaugural.zip.
[nltk_data] | Downloading package movie_reviews to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping corpora/movie_reviews.zip.
[nltk_data] | Downloading package nps_chat to /root/nltk_data...
[nltk_data] | Unzipping corpora/nps_chat.zip.
[nltk_data] | Downloading package names to /root/nltk_data...
[nltk_data] | Unzipping corpora/names.zip.
[nltk_data] | Downloading package ppattach to /root/nltk_data...
[nltk_data] | Unzipping corpora/ppattach.zip.
[nltk_data] | Downloading package reuters to /root/nltk_data...
[nltk_data] | Downloading package senseval to /root/nltk_data...
[nltk_data] | Unzipping corpora/senseval.zip.
[nltk_data] | Downloading package state_union to /root/nltk_data...
[nltk_data] | Unzipping corpora/state_union.zip.
[nltk_data] | Downloading package stopwords to /root/nltk_data...
[nltk_data] | Unzipping corpora/stopwords.zip.
[nltk_data] | Downloading package swadesh to /root/nltk_data...
[nltk_data] | Unzipping corpora/swadesh.zip.
[nltk_data] | Downloading package timit to /root/nltk_data...
[nltk_data] | Unzipping corpora/timit.zip.
[nltk_data] | Downloading package treebank to /root/nltk_data...
[nltk_data] | Unzipping corpora/treebank.zip.
[nltk_data] | Downloading package toolbox to /root/nltk_data...
[nltk_data] | Unzipping corpora/toolbox.zip.
[nltk_data] | Downloading package udhr to /root/nltk_data...
[nltk_data] | Unzipping corpora/udhr.zip.
[nltk_data] | Downloading package udhr2 to /root/nltk_data...
[nltk_data] | Unzipping corpora/udhr2.zip.
[nltk_data] | Downloading package unicode_samples to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping corpora/unicode_samples.zip.
[nltk_data] | Downloading package webtext to /root/nltk_data...
[nltk_data] | Unzipping corpora/webtext.zip.
[nltk_data] | Downloading package wordnet to /root/nltk_data...
[nltk_data] | Unzipping corpora/wordnet.zip.
[nltk_data] | Downloading package wordnet_ic to /root/nltk_data...
[nltk_data] | Unzipping corpora/wordnet_ic.zip.
[nltk_data] | Downloading package words to /root/nltk_data...
[nltk_data] | Unzipping corpora/words.zip.
[nltk_data] | Downloading package maxent_treebank_pos_tagger to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping taggers/maxent_treebank_pos_tagger.zip.
[nltk_data] | Downloading package maxent_ne_chunker to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping chunkers/maxent_ne_chunker.zip.
[nltk_data] | Downloading package universal_tagset to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping taggers/universal_tagset.zip.
[nltk_data] | Downloading package punkt to /root/nltk_data...
[nltk_data] | Unzipping tokenizers/punkt.zip.
[nltk_data] | Downloading package book_grammars to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping grammars/book_grammars.zip.
[nltk_data] | Downloading package city_database to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping corpora/city_database.zip.
[nltk_data] | Downloading package tagsets to /root/nltk_data...
[nltk_data] | Unzipping help/tagsets.zip.
[nltk_data] | Downloading package panlex_swadesh to
[nltk_data] | /root/nltk_data...
[nltk_data] | Downloading package averaged_perceptron_tagger to
[nltk_data] | /root/nltk_data...
[nltk_data] | Unzipping taggers/averaged_perceptron_tagger.zip.
[nltk_data] |
[nltk_data] Done downloading collection book
tweets = pd.read_csv('/work/stocktweets/tweets_labelled_09042020_16072020.csv', sep=';')
tweets
idint64
11 - 938084
created_atobject
2020-04-15 01:03:46+00:000%
2020-06-25 06:20:06+00:000%
4998 others100%
0
77522
2020-04-15 01:03:46+00:00
1
661634
2020-06-25 06:20:06+00:00
2
413231
2020-06-04 15:41:45+00:00
3
760262
2020-07-03 19:39:35+00:00
4
830153
2020-07-09 14:39:14+00:00
5
27027
2020-04-12 21:52:56+00:00
6
472959
2020-06-09 05:23:06+00:00
7
392845
2020-06-02 01:12:29+00:00
8
313771
2020-05-07 04:58:41+00:00
9
267894
2020-05-04 15:16:29+00:00
tweets.head()
idint64
created_atobject
0
77522
2020-04-15 01:03:46+00:00
1
661634
2020-06-25 06:20:06+00:00
2
413231
2020-06-04 15:41:45+00:00
3
760262
2020-07-03 19:39:35+00:00
4
830153
2020-07-09 14:39:14+00:00
tweets['text_normalized'] = tweets.text.str.lower()
from nltk.tokenize import word_tokenize
tweets['words'] = tweets['text_normalized'].apply(word_tokenize)
tweets
idint64
11 - 938084
created_atobject
2020-04-15 01:03:46+00:000%
2020-06-25 06:20:06+00:000%
4998 others100%
0
77522
2020-04-15 01:03:46+00:00
1
661634
2020-06-25 06:20:06+00:00
2
413231
2020-06-04 15:41:45+00:00
3
760262
2020-07-03 19:39:35+00:00
4
830153
2020-07-09 14:39:14+00:00
5
27027
2020-04-12 21:52:56+00:00
6
472959
2020-06-09 05:23:06+00:00
7
392845
2020-06-02 01:12:29+00:00
8
313771
2020-05-07 04:58:41+00:00
9
267894
2020-05-04 15:16:29+00:00
tweets['types'] = tweets['words'].apply(set)
tweets['ttr'] = tweets['types'].apply(len)/tweets['words'].apply(len)
tweets
idint64
11 - 938084
created_atobject
2020-04-15 01:03:46+00:000%
2020-06-25 06:20:06+00:000%
4998 others100%
0
77522
2020-04-15 01:03:46+00:00
1
661634
2020-06-25 06:20:06+00:00
2
413231
2020-06-04 15:41:45+00:00
3
760262
2020-07-03 19:39:35+00:00
4
830153
2020-07-09 14:39:14+00:00
5
27027
2020-04-12 21:52:56+00:00
6
472959
2020-06-09 05:23:06+00:00
7
392845
2020-06-02 01:12:29+00:00
8
313771
2020-05-07 04:58:41+00:00
9
267894
2020-05-04 15:16:29+00:00
'/root/venv/bin/python -m pip install --upgrade pip'
from ntlk.corpus import stopwords
STOPWORDS = stopwords.words('english')
def remove_stopwords(words):
return [word for word in words if word not in STOPWORDS]
tweets['non_stopwords'] = tweets['words'].apply(remove_stopwords)
tweets['non_stopwords_count'] = tweets['non_stopwords'].apply(len)
ModuleNotFoundError: No module named 'ntlk'
tweets['pos'] = tweets['non_stopwords'].apply(nltk.pos_tag)
KeyError: 'non_stopwords'