# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from decimal import Decimal
from traceback import print_exc
from twip.constant import DATA_PATH
from pug.nlp.util import dict2obj
import matplotlib
import os
import pandas as pd
import pandas_profiling
# coding: utf-8

# In[39]:

from __future__ import division, print_function, absolute_import

get_ipython().magic(u'matplotlib inline')



# you really want to be efficient about RAM, so user iter and itertools
# from itertools import izip

np = pd.np


# In[2]:

# this should load 100k tweets in about a minute
print('Loading previously "cleaned" tweets (could take a minute or so)...')
df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), index_col='id', compression='gzip',
                 quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
print('Loaded {} tweets.'.format(len(df)))


# In[18]:

print('df.describe() stats:')
short_desc = df.describe()
for col, stats in short_desc.T.iterrows():

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/99 WIP -- Data Exploration.py
from past.builtins import basestring
from decimal import Decimal
from traceback import print_exc
from twip.constant import DATA_PATH
from pug.nlp.util import dict2obj
import matplotlib
import os
import pandas as pd
import pandas_profiling

# coding: utf-8

# In[39]:

from __future__ import division, print_function, absolute_import

get_ipython().magic(u'matplotlib inline')



# you really want to be efficient about RAM, so user iter and itertools
# from itertools import izip

np = pd.np


# In[2]:

# this should load 100k tweets in about a minute
print('Loading previously "cleaned" tweets (could take a minute or so)...')
df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), index_col='id', compression='gzip',
                 quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
print('Loaded {} tweets.'.format(len(df)))


# In[18]:

print('df.describe() stats:')
short_desc = df.describe()
for col, stats in short_desc.T.iterrows():

    print('{} ({})'.format(col, df[col].dtype if isinstance(df[col], pd.Series) else type(df[col])))