# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
from pug.nlp.regex import url
import os
import pandas as pd
import string
import matplotlib
import gzip
display(HTML("<style>.container { width:100% !important; }</style>"))
pd.set_option('display.max_rows', 6)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 500)


# In[3]:

df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), low_memory=False)


# In[4]:

df = df.drop_duplicates('id', keep='last')[['id', 'id_str', 'text']]
df.id == df.id_str
(df.id != df.id_str).sum()


# In[5]:

df = df[['id', 'text']]


# In[6]:

df.text


# In[8]:


# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/07 Features -- Text.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
from pug.nlp.regex import url
import os
import pandas as pd
import string
import matplotlib
import gzip

display(HTML("<style>.container { width:100% !important; }</style>"))
pd.set_option('display.max_rows', 6)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 500)


# In[3]:

df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), low_memory=False)


# In[4]:

df = df.drop_duplicates('id', keep='last')[['id', 'id_str', 'text']]
df.id == df.id_str
(df.id != df.id_str).sum()


# In[5]:

df = df[['id', 'text']]


# In[6]:

df.text


# In[8]:


df['tokens'] = df.text.str.split()