<commit_before>#!/usr/bin/env python
"""
Download bz2 files from list and upload to Disco Distributed File System.
"""

from __future__ import print_function
import argparse
import os
import glob
import sys
import urllib2
import bz2
import pandas as pd
from disco.ddfs import DDFS

def create_df_concat(fcsv_list):
    """
    Read in a list of CSV files and combine into a single dataframe.
    CSV file names are used as top level of heirarchical index.
    """
    df_dict = {}
    for fcsv in fcsv_list:
        df = csv_to_df(fcsv=fcsv)
        fcsv_basename = os.path.basename(fcsv)
        df_dict[fcsv_basename] = df
    df_concat = pd.concat(df_dict)
    return df_concat

def csv_to_df(fcsv):
    """
    Read a CSV file and return as a dataframe.
    Ignore comments starting with #.
    """
    if not os.path.isfile(fcsv):
        raise IOError(("File does not exist: {fname}").format(fname=fcsv))
    (fcsv_base, ext) = os.path.splitext(fcsv)
    if ext != '.csv':
        raise IOError(("File extension not '.csv': {fname}").format(fname=fcsv))
    fcsv_nocmts = fcsv_base + '_temp' + ext
    with open(fcsv, 'r') as fcmts:
        # Make a temporary file without comments.
        with open(fcsv_nocmts, 'w') as fnocmts:
            for line in fcmts:
                if line.startswith('#'):
                    continue
                else:
                    fnocmts.write(line)
    df = pd.read_csv(fcsv_nocmts, sep=',')
    os.remove(fcsv_nocmts)
    return df

def download(url, fout="download.out"):
    """
    Download a file from a URL.
    """
    # From http://stackoverflow.com/questions/4028697
    # /how-do-i-download-a-zip-file-in-python-using-urllib2
    f_url = urllib2.urlopen(url)
    with open(fout, 'wb') as fo:
        fo.write(f_url.read())
    return None

def decompress_and_partition(fbz2, fout="decompress.out"):
    """
    Decompress bz2 files and insert newlines every 100 KB.
    Due to a bug in disco v0.4.4 for uploading, pushing a long record as a blob corrupts the existing tag.
    Chunk does not work on records over 1MB.
    See: https://groups.google.com/forum/#!searchin/disco-dev/push$20chunk/disco-dev/i9LiNiLEQ7k/95fX2sC-dtQJ
    """
    (base, ext) = os.path.splitext(fbz2)
    if ext != '.bz2':
        raise IOError(("File extension not '.bz2': {fname}").format(fname=fbz2))
    # Read large file incrementally and insert newlines every 100 KB. From:
    # http://stackoverflow.com/questions/16963352/decompress-bz2-files
    # http://bob.ippoli.to/archives/2005/06/14/python-iterators-and-sentinel-values/
    with open(fout, 'wb') as fo, bz2.BZ2File(fbz2, 'rb') as fb:
        for data in iter(lambda : fb.read(100*1024), b''):
            fo.write(data)
            fo.write(b"\n")
    return None

class ErrMsg(object):
    """
    Print custom error messages.
    """
    _err_count = 0

    def __init__(self):<commit_msg>making check_filetags<commit_after>#!/usr/bin/env python
"""
Download bz2 files from list and upload to Disco Distributed File System.
"""

from __future__ import print_function
import argparse
import os
import glob
import sys
import urllib2
import bz2
import pandas as pd
from disco.ddfs import DDFS

def create_df_concat(fcsv_list):
    """
    Read in a list of CSV files and combine into a single dataframe.
    CSV file names are used as top level of heirarchical index.
    """
    df_dict = {}
    for fcsv in fcsv_list:
        df = csv_to_df(fcsv=fcsv)
        fcsv_basename = os.path.basename(fcsv)
        df_dict[fcsv_basename] = df
    df_concat = pd.concat(df_dict)
    return df_concat

def csv_to_df(fcsv):
    """
    Read a CSV file and return as a dataframe.
    Ignore comments starting with #.
    """
    if not os.path.isfile(fcsv):
        raise IOError(("File does not exist: {fname}").format(fname=fcsv))
    (fcsv_base, ext) = os.path.splitext(fcsv)
    if ext != '.csv':
        raise IOError(("File extension not '.csv': {fname}").format(fname=fcsv))
    fcsv_nocmts = fcsv_base + '_temp' + ext
    with open(fcsv, 'r') as fcmts:
        # Make a temporary file without comments.
        with open(fcsv_nocmts, 'w') as fnocmts:
            for line in fcmts:
                if line.startswith('#'):
                    continue
                else:
                    fnocmts.write(line)
    df = pd.read_csv(fcsv_nocmts, sep=',')
    os.remove(fcsv_nocmts)
    return df

def download(url, fout="download.out"):
    """
    Download a file from a URL.
    """
    # From http://stackoverflow.com/questions/4028697
    # /how-do-i-download-a-zip-file-in-python-using-urllib2
    f_url = urllib2.urlopen(url)
    with open(fout, 'wb') as fo:
        fo.write(f_url.read())
    return None

def decompress_and_partition(fbz2, fout="decompress.out"):
    """
    Decompress bz2 files and insert newlines every 100 KB.
    Due to a bug in disco v0.4.4 for uploading, pushing a long record as a blob corrupts the existing tag.
    Chunk does not work on records over 1MB.
    See: https://groups.google.com/forum/#!searchin/disco-dev/push$20chunk/disco-dev/i9LiNiLEQ7k/95fX2sC-dtQJ
    """
    (base, ext) = os.path.splitext(fbz2)
    if ext != '.bz2':
        raise IOError(("File extension not '.bz2': {fname}").format(fname=fbz2))
    # Read large file incrementally and insert newlines every 100 KB. From:
    # http://stackoverflow.com/questions/16963352/decompress-bz2-files
    # http://bob.ippoli.to/archives/2005/06/14/python-iterators-and-sentinel-values/
    with open(fout, 'wb') as fo, bz2.BZ2File(fbz2, 'rb') as fb:
        for data in iter(lambda : fb.read(100*1024), b''):
            fo.write(data)
            fo.write(b'\n')
    return None

class ErrMsg(object):
    """
    Print custom error messages.
    """
    _err_count = 0

    def __init__(self):
{'fraction_non_alphanumeric': 0.11904761904761904, 'fraction_numerical': 0.021739130434782608, 'mean_word_length': 2.701787994891443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5583565', 'n_tokens_mistral': 2125, 'n_tokens_neox': 2020, 'n_words': 536}