# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


import os
import pandas as pd
import progressbar
from collections import Counter
from twip.constant import DATA_PATH
from pug.nlp.segmentation import Tokenizer
            pbar.update(pbar_i)
        if pbar_i > num_tweets:
            break
        text = row.text if isinstance(row.text, basestring) and row.text else ''
        counts[twid] = pd.Series(Counter(segment_words(text)), name=twid, dtype='uint16')
        # counts = pd.concat([counts, pd.Series(Counter(segment_words(text)), name=twid)], axis=1)
        stats += [[counts[twid].sum(), np.sum(np.array(counts[twid] > 0)), len(text)]]
    if verbosity > 0:
        pbar.finish()
    stats = pd.DataFrame(stats, index=df.index[:len(stats)], columns=['text_num_words', 'text_num_unique', 'text_len'])
    df = pd.concat([df.iloc[:num_tweets], stats], axis=1)
    if verbosity:
        pbar.finish()
    return df, counts.T


def run(num_tweets=1000000, verbosity=1):
    print('Loading previously cleaned tweets...')
    # the round-trip to disk cleans up encoding issues so encoding option no longer needs to be specified and gzip 
    df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), index_col='id', compression='gzip',
                      quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
    print('Found {} tweets.'.format(len(df)))
    print('Loading previously compiled vocab...')
    # the round-trip to disk cleans up encoding issues so encoding option no longer needs to be specified and gzip 
    try:
        tfdf = pd.read_csv(os.path.join(DATA_PATH, 'tweet_vocab.csv.gz'), index_col=0, compression='gzip',
                          quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
        assert(len(tfdf) > 10000)
    except:
        tfdf = tf_df(df, num_tweets=num_tweets, verbosity=verbosity)

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: twip/scripts/generate.py
import os
import pandas as pd
import progressbar
from collections import Counter
from twip.constant import DATA_PATH
from pug.nlp.segmentation import Tokenizer

            pbar.update(pbar_i)
        if pbar_i > num_tweets:
            break
        text = row.text if isinstance(row.text, basestring) and row.text else ''
        counts[twid] = pd.Series(Counter(segment_words(text)), name=twid, dtype='uint16')
        # counts = pd.concat([counts, pd.Series(Counter(segment_words(text)), name=twid)], axis=1)
        stats += [[counts[twid].sum(), np.sum(np.array(counts[twid] > 0)), len(text)]]
    if verbosity > 0:
        pbar.finish()
    stats = pd.DataFrame(stats, index=df.index[:len(stats)], columns=['text_num_words', 'text_num_unique', 'text_len'])
    df = pd.concat([df.iloc[:num_tweets], stats], axis=1)
    if verbosity:
        pbar.finish()
    return df, counts.T


def run(num_tweets=1000000, verbosity=1):
    print('Loading previously cleaned tweets...')
    # the round-trip to disk cleans up encoding issues so encoding option no longer needs to be specified and gzip 
    df = pd.read_csv(os.path.join(DATA_PATH, 'cleaned_tweets.csv.gz'), index_col='id', compression='gzip',
                      quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
    print('Found {} tweets.'.format(len(df)))
    print('Loading previously compiled vocab...')
    # the round-trip to disk cleans up encoding issues so encoding option no longer needs to be specified and gzip 
    try:
        tfdf = pd.read_csv(os.path.join(DATA_PATH, 'tweet_vocab.csv.gz'), index_col=0, compression='gzip',
                          quotechar='"', quoting=pd.io.common.csv.QUOTE_NONNUMERIC, low_memory=False)
        assert(len(tfdf) > 10000)
    except:
        tfdf = tf_df(df, num_tweets=num_tweets, verbosity=verbosity)

        tfdf.to_csv(os.path.join(DATA_PATH, 'tweet_vocab.csv.gz'), compression='gzip',