# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from matplotlib import pyplot as plt
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
import os
import pandas as pd
import matplotlib
import gzip

df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), low_memory=False)
rawlen = len(df)
df.drop_duplicates('id_str', keep='last', inplace=True)
rawlen - len(df)


# In[5]:

# df.get_
df.get_dtype_counts()


# In[9]:

dtypes = pd.Series([df[v].dtype for v in df.columns], index=df.columns)
dtypes


# In[6]:

mask = [col for col in df.columns if df[col].dtype in (bool, float, np.dtype('int64'))]  # don't forget bool and int64 (not int)!
mask
numbers = df[mask]
numbers


# In[8]:

with gzip.open(os.path.join(DATA_PATH, 'numbers.csv.gz'), 'wb') as f:

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/06 Features -- Numeric.py
from past.builtins import basestring
from matplotlib import pyplot as plt
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
import os
import pandas as pd
import matplotlib
import gzip


df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), low_memory=False)
rawlen = len(df)
df.drop_duplicates('id_str', keep='last', inplace=True)
rawlen - len(df)


# In[5]:

# df.get_
df.get_dtype_counts()


# In[9]:

dtypes = pd.Series([df[v].dtype for v in df.columns], index=df.columns)
dtypes


# In[6]:

mask = [col for col in df.columns if df[col].dtype in (bool, float, np.dtype('int64'))]  # don't forget bool and int64 (not int)!
mask
numbers = df[mask]
numbers


# In[8]:

with gzip.open(os.path.join(DATA_PATH, 'numbers.csv.gz'), 'wb') as f:

    numbers.to_csv(os.path.join(DATA_PATH, 'numbers.csv.gz'), encoding='utf-8')