# Path: twip/futil.py
# def find_files(path='', ext='', level=None, typ=list, dirs=False, files=True, verbosity=0):
#     """ Recursively find all files in the indicated directory
# 
#     Filter by the indicated file name extension (ext)
# 
#     Args:
#       path (str):  Root/base path to search.
#       ext (str):   File name extension. Only file paths that ".endswith()" this string will be returned
#       level (int, optional): Depth of file tree to halt recursion at.
#         None = full recursion to as deep as it goes
#         0 = nonrecursive, just provide a list of files at the root level of the tree
#         1 = one level of depth deeper in the tree
#       typ (type):  output type (default: list). if a mapping type is provided the keys will be the full paths (unique)
#       dirs (bool):  Whether to yield dir paths along with file paths (default: False)
#       files (bool): Whether to yield file paths (default: True)
#         `dirs=True`, `files=False` is equivalent to `ls -d`
# 
#     Returns:
#       list of dicts: dict keys are { 'path', 'name', 'bytes', 'created', 'modified', 'accessed', 'permissions' }
#         path (str): Full, absolute paths to file beneath the indicated directory and ending with `ext`
#         name (str): File name only (everythin after the last slash in the path)
#         size (int): File size in bytes
#         created (datetime): File creation timestamp from file system
#         modified (datetime): File modification timestamp from file system
#         accessed (datetime): File access timestamp from file system
#         permissions (int): File permissions bytes as a chown-style integer with a maximum of 4 digits
#         type (str): One of 'file', 'dir', 'symlink->file', 'symlink->dir', 'symlink->broken'
#           e.g.: 777 or 1755
# 
#     Examples:
#       >>> 'util.py' in [d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0)]
#       True
#       >>> (d for d in find_files(os.path.dirname(__file__), ext='.py') if d['name'] == 'util.py').next()['size'] > 1000
#       True
# 
#       There should be an __init__ file in the same directory as this script.
#       And it should be at the top of the list.
#       >>> sorted(d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0))[0]
#       '__init__.py'
#       >>> all(d['type'] in ('file','dir','symlink->file','symlink->dir','mount-point->file','mount-point->dir','block-device',
#                             'symlink->broken','pipe','special','socket','unknown') for d in find_files(level=1, files=True, dirs=True))
#       True
#       >>> os.path.join(os.path.dirname(__file__), '__init__.py') in find_files(
#       ... os.path.dirname(__file__), ext='.py', level=0, typ=dict)
#       True
#     """
#     gen = generate_files(path, ext=ext, level=level, dirs=dirs, files=files, verbosity=verbosity)
#     if isinstance(typ(), collections.Mapping):
#         return typ((ff['path'], ff) for ff in gen)
#     elif typ is not None:
#         return typ(gen)
#     else:
#         return gen
# 
# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from future.utils import viewitems  # noqa
from builtins import str  # noqa
from past.builtins import basestring  # noqa
    from itertools import izip as zip
from twip.futil import find_files
from twip.constant import DATA_PATH
from twip import __version__
import os
import re
import json
import logging
import time
import gzip
import pandas as pd
import progressbar
import argparse
import sys
            save_tweets(df_all, path=path, filename='tmp.csv')
        if pbar:
            pbar.update(loaded_size / 1e6)
    print(len(df_all))
    bigger_better_cols = [c for c in df_all.columns if c.endswith('_at') or '_count' in c]
    print(bigger_better_cols)
    print(all(c in df_all.columns for c in bigger_better_cols))
    df_all = df_all.sort_values(by=bigger_better_cols, inplace=False)
    hashable_cols = [c for c in df_all.columns if df_all[c].dtype not in (list,)]
    df_all = df_all.drop_duplicates(subset=hashable_cols, inplace=False)
    if pbar:
        pbar.finish()
    log.info('Loaded {} unique tweets.'.format(len(df_all)))
    return df_all


def drop_nan_columns(df, thresh=325):
    """Drop columns that are mostly NaNs

    Excel files can only have 256 columns, so you may have to drop a lot in order to get down to this
    """
    if thresh < 1:
        thresh = int(thresh * df)
    return df.dropna(axis=1, thresh=thresh, inplace=False)


def drop_columns(df, columns=u'common_columns.json'):
    # df_all = drop_columns(df_all)
    # common_columns = json.dump(list(df_all.columns), open(os.path.join(DATA_PATH, 'common_columns.json'), 'w'), indent=0)
    if isinstance(columns, str):

# Path: twip/futil.py
# def find_files(path='', ext='', level=None, typ=list, dirs=False, files=True, verbosity=0):
#     """ Recursively find all files in the indicated directory
# 
#     Filter by the indicated file name extension (ext)
# 
#     Args:
#       path (str):  Root/base path to search.
#       ext (str):   File name extension. Only file paths that ".endswith()" this string will be returned
#       level (int, optional): Depth of file tree to halt recursion at.
#         None = full recursion to as deep as it goes
#         0 = nonrecursive, just provide a list of files at the root level of the tree
#         1 = one level of depth deeper in the tree
#       typ (type):  output type (default: list). if a mapping type is provided the keys will be the full paths (unique)
#       dirs (bool):  Whether to yield dir paths along with file paths (default: False)
#       files (bool): Whether to yield file paths (default: True)
#         `dirs=True`, `files=False` is equivalent to `ls -d`
# 
#     Returns:
#       list of dicts: dict keys are { 'path', 'name', 'bytes', 'created', 'modified', 'accessed', 'permissions' }
#         path (str): Full, absolute paths to file beneath the indicated directory and ending with `ext`
#         name (str): File name only (everythin after the last slash in the path)
#         size (int): File size in bytes
#         created (datetime): File creation timestamp from file system
#         modified (datetime): File modification timestamp from file system
#         accessed (datetime): File access timestamp from file system
#         permissions (int): File permissions bytes as a chown-style integer with a maximum of 4 digits
#         type (str): One of 'file', 'dir', 'symlink->file', 'symlink->dir', 'symlink->broken'
#           e.g.: 777 or 1755
# 
#     Examples:
#       >>> 'util.py' in [d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0)]
#       True
#       >>> (d for d in find_files(os.path.dirname(__file__), ext='.py') if d['name'] == 'util.py').next()['size'] > 1000
#       True
# 
#       There should be an __init__ file in the same directory as this script.
#       And it should be at the top of the list.
#       >>> sorted(d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0))[0]
#       '__init__.py'
#       >>> all(d['type'] in ('file','dir','symlink->file','symlink->dir','mount-point->file','mount-point->dir','block-device',
#                             'symlink->broken','pipe','special','socket','unknown') for d in find_files(level=1, files=True, dirs=True))
#       True
#       >>> os.path.join(os.path.dirname(__file__), '__init__.py') in find_files(
#       ... os.path.dirname(__file__), ext='.py', level=0, typ=dict)
#       True
#     """
#     gen = generate_files(path, ext=ext, level=level, dirs=dirs, files=files, verbosity=verbosity)
#     if isinstance(typ(), collections.Mapping):
#         return typ((ff['path'], ff) for ff in gen)
#     elif typ is not None:
#         return typ(gen)
#     else:
#         return gen
# 
# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: twip/scripts/cat_tweets.py
from future.utils import viewitems  # noqa
from builtins import str  # noqa
from past.builtins import basestring  # noqa
    from itertools import izip as zip
from twip.futil import find_files
from twip.constant import DATA_PATH
from twip import __version__
import os
import re
import json
import logging
import time
import gzip
import pandas as pd
import progressbar
import argparse
import sys

            save_tweets(df_all, path=path, filename='tmp.csv')
        if pbar:
            pbar.update(loaded_size / 1e6)
    print(len(df_all))
    bigger_better_cols = [c for c in df_all.columns if c.endswith('_at') or '_count' in c]
    print(bigger_better_cols)
    print(all(c in df_all.columns for c in bigger_better_cols))
    df_all = df_all.sort_values(by=bigger_better_cols, inplace=False)
    hashable_cols = [c for c in df_all.columns if df_all[c].dtype not in (list,)]
    df_all = df_all.drop_duplicates(subset=hashable_cols, inplace=False)
    if pbar:
        pbar.finish()
    log.info('Loaded {} unique tweets.'.format(len(df_all)))
    return df_all


def drop_nan_columns(df, thresh=325):
    """Drop columns that are mostly NaNs

    Excel files can only have 256 columns, so you may have to drop a lot in order to get down to this
    """
    if thresh < 1:
        thresh = int(thresh * df)
    return df.dropna(axis=1, thresh=thresh, inplace=False)


def drop_columns(df, columns=u'common_columns.json'):
    # df_all = drop_columns(df_all)
    # common_columns = json.dump(list(df_all.columns), open(os.path.join(DATA_PATH, 'common_columns.json'), 'w'), indent=0)
    if isinstance(columns, str):

        columns = json.load(open(os.path.join(DATA_PATH, columns), 'r'))