<commit_before>import config
import os
import re
import sys, urllib
import shutil
import tarfile

def output(message, rpad=0, rpad_char='#'):
    s = "#####J> %s" % (message,)
    pn = rpad - len(s)
    if pn < 0:
        pn = 0
    p = pn * rpad_char
        
    print "%s %s" % (s,p)

def error(message):
    raise RuntimeError('!!!!! %s' % (message,))

def urlget(url):
    """Simple method to retrieve URL.  It will get the file in the current
    directory.
    """
    
    def reporthook(blocknum, blocksize, totalsize):
        current_size = blocknum * blocksize
        current_size_kb = int(current_size / 1024.0)
        sys.stdout.write(
            '% 4.0f %% (%d Kbytes) downloaded\r' %
            (current_size / float(totalsize) * 100.0, current_size_kb))

    i = url.rfind('/')
    filename = url[i+1:]
    print url, "->", filename
    if os.path.exists(filename):
        output("%s already present, skipping download." % (filename,))

    else:
        urllib.urlretrieve(url, filename, reporthook)
        sys.stdout.write("\n")
        output("Download complete.")

    return filename


def goto_archive():
    os.chdir(config.archive_dir)

def goto_build():
    os.chdir(config.build_dir)

def unpack_build(archive_filename):
    """Unpack given archive_filename in build directory.
    """
    goto_build()

    if archive_filename.lower().endswith('bz2'):
        m = 'r|bz2'
    else:
        m = 'r|gz'
        
    tar = tarfile.open(archive_filename, m)

    # extractall is from python 2.5 onwards
    # tar.extractall()
    # we use a form that works on previous versions as well
    for tarinfo in tar:
        tar.extract(tarinfo)
        
    tar.close()

def re_sub_filter_file(repls, filename):
    """Given a list of repls (tuples with regular expresions and
    replacement patterns that are used as the first and second params
    of re.sub), filter filename line by line.

    A backup of the file is made to filename.orig.
    """

    newfilename = '%s.new' % (filename,)
    origfilename = '%s.orig' % (filename,)

    shutil.copyfile(filename, origfilename)

    ifile = file(filename)
    ofile = file(newfilename, 'w')

    for l in ifile:
        for r in repls:
            l = re.sub(r[0], r[1], l)

        ofile.write(l)
<commit_msg>Output name of filename as we extract it from a  tarfile.<commit_after>import config
import os
import re
import sys, urllib
import shutil
import tarfile

def output(message, rpad=0, rpad_char='#'):
    s = "#####J> %s" % (message,)
    pn = rpad - len(s)
    if pn < 0:
        pn = 0
    p = pn * rpad_char
        
    print "%s %s" % (s,p)

def error(message):
    raise RuntimeError('!!!!! %s' % (message,))

def urlget(url):
    """Simple method to retrieve URL.  It will get the file in the current
    directory.
    """
    
    def reporthook(blocknum, blocksize, totalsize):
        current_size = blocknum * blocksize
        current_size_kb = int(current_size / 1024.0)
        sys.stdout.write(
            '% 4.0f %% (%d Kbytes) downloaded\r' %
            (current_size / float(totalsize) * 100.0, current_size_kb))

    i = url.rfind('/')
    filename = url[i+1:]
    print url, "->", filename
    if os.path.exists(filename):
        output("%s already present, skipping download." % (filename,))

    else:
        urllib.urlretrieve(url, filename, reporthook)
        sys.stdout.write("\n")
        output("Download complete.")

    return filename


def goto_archive():
    os.chdir(config.archive_dir)

def goto_build():
    os.chdir(config.build_dir)

def unpack_build(archive_filename):
    """Unpack given archive_filename in build directory.
    """
    goto_build()

    if archive_filename.lower().endswith('bz2'):
        m = 'r|bz2'
    else:
        m = 'r|gz'
        
    tar = tarfile.open(archive_filename, m)

    # extractall is from python 2.5 onwards
    # tar.extractall()
    # we use a form that works on previous versions as well
    for tarinfo in tar:
        print tarinfo.name
        tar.extract(tarinfo)
        
    tar.close()

def re_sub_filter_file(repls, filename):
    """Given a list of repls (tuples with regular expresions and
    replacement patterns that are used as the first and second params
    of re.sub), filter filename line by line.

    A backup of the file is made to filename.orig.
    """

    newfilename = '%s.new' % (filename,)
    origfilename = '%s.orig' % (filename,)

    shutil.copyfile(filename, origfilename)

    ifile = file(filename)
    ofile = file(newfilename, 'w')

    for l in ifile:
        for r in repls:
            l = re.sub(r[0], r[1], l)

        ofile.write(l)

{'fraction_non_alphanumeric': 0.12346741234674123, 'fraction_numerical': 0.00903420090342009, 'mean_word_length': 2.512084592145015, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '544963', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1505, 'n_words': 444}