# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from decimal import Decimal
from traceback import print_exc
from twip.constant import DATA_PATH
from pug.nlp.util import make_name
from pug.nlp.scrape import transcode_unicode
import os
import progressbar
import pandas as pd
            # series = df[col].copy()
            # for i, value in series.iteritems():   

            #     if isinstance(value, basestring):
            #         series[i] = str(value.encode(encoding))
            # df[col] = series
    if verbosity:
        pbar.finish()
    return df

def run(verbosity=1):
    """Load all_tweets.csv and run normalize, dropna, encode before dumping to cleaned_tweets.csv.gz

    Many columns have "mixed type" to `read_csv` should set `low_memory` to False to ensure they're loaded accurately
    >>> df2 = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), index_col='id', compression='gzip', quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
    /home/hobs/.virtualenvs/twip/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2723:
    DtypeWarning: Columns (74,82,84,105,114,115,116,117,118,119,120,121,122,123,125,126,127,128,129,130,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,149,150,151,152,199,200,202,203,210,211,214,215,223,231,232,238) have mixed types. Specify dtype option on import or set low_memory=False.
       interactivity=interactivity, compiler=compiler, result=result)
    """
    filepath = os.path.join(DATA_PATH, 'all_tweets.csv')
    # this should load 100k tweets in about a minute
    # check the file size and estimate load time from that (see scritps/cat_tweets.py)
    print('Loading tweets from {} (could take a minute or so)...'.format(filepath))
    df = pd.read_csv(filepath, encoding='utf-8', engine='python')
    if 'id' in df.columns:
        df = df.set_index('id')
    df = normalize(df)
    df = dropna(df)
    df = encode(df, verbosity=verbosity)
    df = clean_labels(df)

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: twip/scripts/clean.py
from past.builtins import basestring
from decimal import Decimal
from traceback import print_exc
from twip.constant import DATA_PATH
from pug.nlp.util import make_name
from pug.nlp.scrape import transcode_unicode
import os
import progressbar
import pandas as pd

            # series = df[col].copy()
            # for i, value in series.iteritems():   

            #     if isinstance(value, basestring):
            #         series[i] = str(value.encode(encoding))
            # df[col] = series
    if verbosity:
        pbar.finish()
    return df

def run(verbosity=1):
    """Load all_tweets.csv and run normalize, dropna, encode before dumping to cleaned_tweets.csv.gz

    Many columns have "mixed type" to `read_csv` should set `low_memory` to False to ensure they're loaded accurately
    >>> df2 = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), index_col='id', compression='gzip', quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
    /home/hobs/.virtualenvs/twip/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2723:
    DtypeWarning: Columns (74,82,84,105,114,115,116,117,118,119,120,121,122,123,125,126,127,128,129,130,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,149,150,151,152,199,200,202,203,210,211,214,215,223,231,232,238) have mixed types. Specify dtype option on import or set low_memory=False.
       interactivity=interactivity, compiler=compiler, result=result)
    """
    filepath = os.path.join(DATA_PATH, 'all_tweets.csv')
    # this should load 100k tweets in about a minute
    # check the file size and estimate load time from that (see scritps/cat_tweets.py)
    print('Loading tweets from {} (could take a minute or so)...'.format(filepath))
    df = pd.read_csv(filepath, encoding='utf-8', engine='python')
    if 'id' in df.columns:
        df = df.set_index('id')
    df = normalize(df)
    df = dropna(df)
    df = encode(df, verbosity=verbosity)
    df = clean_labels(df)

    df.to_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), compression='gzip',