# Path: twip/futil.py
# def find_files(path='', ext='', level=None, typ=list, dirs=False, files=True, verbosity=0):
#     """ Recursively find all files in the indicated directory
# 
#     Filter by the indicated file name extension (ext)
# 
#     Args:
#       path (str):  Root/base path to search.
#       ext (str):   File name extension. Only file paths that ".endswith()" this string will be returned
#       level (int, optional): Depth of file tree to halt recursion at.
#         None = full recursion to as deep as it goes
#         0 = nonrecursive, just provide a list of files at the root level of the tree
#         1 = one level of depth deeper in the tree
#       typ (type):  output type (default: list). if a mapping type is provided the keys will be the full paths (unique)
#       dirs (bool):  Whether to yield dir paths along with file paths (default: False)
#       files (bool): Whether to yield file paths (default: True)
#         `dirs=True`, `files=False` is equivalent to `ls -d`
# 
#     Returns:
#       list of dicts: dict keys are { 'path', 'name', 'bytes', 'created', 'modified', 'accessed', 'permissions' }
#         path (str): Full, absolute paths to file beneath the indicated directory and ending with `ext`
#         name (str): File name only (everythin after the last slash in the path)
#         size (int): File size in bytes
#         created (datetime): File creation timestamp from file system
#         modified (datetime): File modification timestamp from file system
#         accessed (datetime): File access timestamp from file system
#         permissions (int): File permissions bytes as a chown-style integer with a maximum of 4 digits
#         type (str): One of 'file', 'dir', 'symlink->file', 'symlink->dir', 'symlink->broken'
#           e.g.: 777 or 1755
# 
#     Examples:
#       >>> 'util.py' in [d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0)]
#       True
#       >>> (d for d in find_files(os.path.dirname(__file__), ext='.py') if d['name'] == 'util.py').next()['size'] > 1000
#       True
# 
#       There should be an __init__ file in the same directory as this script.
#       And it should be at the top of the list.
#       >>> sorted(d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0))[0]
#       '__init__.py'
#       >>> all(d['type'] in ('file','dir','symlink->file','symlink->dir','mount-point->file','mount-point->dir','block-device',
#                             'symlink->broken','pipe','special','socket','unknown') for d in find_files(level=1, files=True, dirs=True))
#       True
#       >>> os.path.join(os.path.dirname(__file__), '__init__.py') in find_files(
#       ... os.path.dirname(__file__), ext='.py', level=0, typ=dict)
#       True
#     """
#     gen = generate_files(path, ext=ext, level=level, dirs=dirs, files=files, verbosity=verbosity)
#     if isinstance(typ(), collections.Mapping):
#         return typ((ff['path'], ff) for ff in gen)
#     elif typ is not None:
#         return typ(gen)
#     else:
#         return gen
# 
# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from future.utils import viewitems  # noqa
from builtins import str  # noqa
from past.builtins import basestring  # noqa
    from itertools import izip as zip
from twip.futil import find_files
from twip.constant import DATA_PATH
from twip import __version__
import os
import re
import json
import logging
import time
import gzip
import pandas as pd
import progressbar
import argparse
import sys
    geo = get_geo(df, path=args.path, filename=args.geofile)
    log.info("Combined {} tweets into a single file {} and set asside {} geo tweets in {}".format(
        len(df), args.tweetfile, len(geo), args.geofile))
    return df, geo


def run():
    """Entry point for command line script"""
    main(sys.argv[1:])


def cat_tweets(filename='all_tweets.json.gz', path=DATA_PATH, ext='.json', save_tmp=False, verbosity=1, numtweets=10000000, ignore_suspicious=True):
    """Find json files that were dumped by tweetget and combine them into a single CSV

    Normalize some (lat/lon)"""
    log.info('Finding {} files in {}...'.format(ext, path))
    meta_files = find_files(path=path, ext=ext)
    meta_files = [meta for meta in meta_files
                  if re.match(r'^[-#@a-z ]*201[5-6]-[0-9]{2}-[0-9]{2}.*', meta['name'])]
    #  '\s[0-9]{2}[:][0-9]{2}[:][0-9]{2}[.][0-9]+[.]json(.gz)?$', meta['name'])]
    log.info('Found {} files that look like tweetget dumps.'.format(len(meta_files)))
    print([mf['name'] for mf in meta_files])
    total_size = sum([meta['size'] for meta in meta_files])
    if verbosity > 0:
        pbar = progressbar.ProgressBar(maxval=(total_size + 1.) / 1e6)
        pbar.start()
    else:
        pbar = None

    loaded_size = 0

# Path: twip/futil.py
# def find_files(path='', ext='', level=None, typ=list, dirs=False, files=True, verbosity=0):
#     """ Recursively find all files in the indicated directory
# 
#     Filter by the indicated file name extension (ext)
# 
#     Args:
#       path (str):  Root/base path to search.
#       ext (str):   File name extension. Only file paths that ".endswith()" this string will be returned
#       level (int, optional): Depth of file tree to halt recursion at.
#         None = full recursion to as deep as it goes
#         0 = nonrecursive, just provide a list of files at the root level of the tree
#         1 = one level of depth deeper in the tree
#       typ (type):  output type (default: list). if a mapping type is provided the keys will be the full paths (unique)
#       dirs (bool):  Whether to yield dir paths along with file paths (default: False)
#       files (bool): Whether to yield file paths (default: True)
#         `dirs=True`, `files=False` is equivalent to `ls -d`
# 
#     Returns:
#       list of dicts: dict keys are { 'path', 'name', 'bytes', 'created', 'modified', 'accessed', 'permissions' }
#         path (str): Full, absolute paths to file beneath the indicated directory and ending with `ext`
#         name (str): File name only (everythin after the last slash in the path)
#         size (int): File size in bytes
#         created (datetime): File creation timestamp from file system
#         modified (datetime): File modification timestamp from file system
#         accessed (datetime): File access timestamp from file system
#         permissions (int): File permissions bytes as a chown-style integer with a maximum of 4 digits
#         type (str): One of 'file', 'dir', 'symlink->file', 'symlink->dir', 'symlink->broken'
#           e.g.: 777 or 1755
# 
#     Examples:
#       >>> 'util.py' in [d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0)]
#       True
#       >>> (d for d in find_files(os.path.dirname(__file__), ext='.py') if d['name'] == 'util.py').next()['size'] > 1000
#       True
# 
#       There should be an __init__ file in the same directory as this script.
#       And it should be at the top of the list.
#       >>> sorted(d['name'] for d in find_files(os.path.dirname(__file__), ext='.py', level=0))[0]
#       '__init__.py'
#       >>> all(d['type'] in ('file','dir','symlink->file','symlink->dir','mount-point->file','mount-point->dir','block-device',
#                             'symlink->broken','pipe','special','socket','unknown') for d in find_files(level=1, files=True, dirs=True))
#       True
#       >>> os.path.join(os.path.dirname(__file__), '__init__.py') in find_files(
#       ... os.path.dirname(__file__), ext='.py', level=0, typ=dict)
#       True
#     """
#     gen = generate_files(path, ext=ext, level=level, dirs=dirs, files=files, verbosity=verbosity)
#     if isinstance(typ(), collections.Mapping):
#         return typ((ff['path'], ff) for ff in gen)
#     elif typ is not None:
#         return typ(gen)
#     else:
#         return gen
# 
# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: twip/scripts/cat_tweets.py
from future.utils import viewitems  # noqa
from builtins import str  # noqa
from past.builtins import basestring  # noqa
    from itertools import izip as zip
from twip.futil import find_files
from twip.constant import DATA_PATH
from twip import __version__
import os
import re
import json
import logging
import time
import gzip
import pandas as pd
import progressbar
import argparse
import sys

    geo = get_geo(df, path=args.path, filename=args.geofile)
    log.info("Combined {} tweets into a single file {} and set asside {} geo tweets in {}".format(
        len(df), args.tweetfile, len(geo), args.geofile))
    return df, geo


def run():
    """Entry point for command line script"""
    main(sys.argv[1:])


def cat_tweets(filename='all_tweets.json.gz', path=DATA_PATH, ext='.json', save_tmp=False, verbosity=1, numtweets=10000000, ignore_suspicious=True):
    """Find json files that were dumped by tweetget and combine them into a single CSV

    Normalize some (lat/lon)"""
    log.info('Finding {} files in {}...'.format(ext, path))
    meta_files = find_files(path=path, ext=ext)
    meta_files = [meta for meta in meta_files
                  if re.match(r'^[-#@a-z ]*201[5-6]-[0-9]{2}-[0-9]{2}.*', meta['name'])]
    #  '\s[0-9]{2}[:][0-9]{2}[:][0-9]{2}[.][0-9]+[.]json(.gz)?$', meta['name'])]
    log.info('Found {} files that look like tweetget dumps.'.format(len(meta_files)))
    print([mf['name'] for mf in meta_files])
    total_size = sum([meta['size'] for meta in meta_files])
    if verbosity > 0:
        pbar = progressbar.ProgressBar(maxval=(total_size + 1.) / 1e6)
        pbar.start()
    else:
        pbar = None

    loaded_size = 0

    df_all = pd.DataFrame()