<commit_before>"""Script to act as a template"""

import os
import sys
import tarfile


from dotsite.paths import makepath


def type_of_tarball(path):
    ext = path.ext
    if ext == '.tar':
        return 'tar'
    if ext in ['.gz', '.bz2']:
        stem, _ = path.splitext()
        if stem.ext == '.tar':
            return ext[1:]
    return None


def is_tarball(path):
    return path.isfile() and type_of_tarball(path) is not None


def directory_tarballs(paths):
    directories = [p for p in paths if p.isdir()]
    result = []
    for directory in directories:
        result.extend([
            path for path in directory.parent.glob(directory.name + '.tar*')
            if is_tarball(path)])
    return result


def find_tarballs(args):
    paths = [makepath(a) for a in args]
    tarballs = [p for p in paths if is_tarball(p)]
    tarballs.extend(directory_tarballs(paths))
    return tarballs


def removable_contents(tarball):
    names = {n.split('/', 1)[0] for n in tarfile.open(tarball).getnames()}
    paths = [makepath(n) for n in names]
    return [p for p in paths if p.isfile() or p.isdir()]


def remove(tarballs):
    for tarball in tarballs:
        for path in removable_contents(tarball):
            if path.isdir():
                print 'rm -rf', path<commit_msg>better naming<commit_after>"""Script to act as a template"""

import os
import sys
import tarfile


from dotsite.paths import makepath


def has_tarball_extension(path):
    ext = path.ext
    if ext == '.tar':
        return True
    if ext in ['.gz', '.bz2']:
        stem, _ = path.splitext()
        if stem.ext == '.tar':
            return True
    return False


def is_tarball(path):
    return path.isfile() and has_tarball_extension(path)


def directory_tarballs(paths):
    directories = [p for p in paths if p.isdir()]
    result = []
    for directory in directories:
        result.extend([
            path for path in directory.parent.glob(directory.name + '.tar*')
            if is_tarball(path)])
    return result


def find_tarballs(args):
    paths = [makepath(a) for a in args]
    tarballs = [p for p in paths if is_tarball(p)]
    tarballs.extend(directory_tarballs(paths))
    return tarballs


def removable_contents(tarball):
    names = {n.split('/', 1)[0] for n in tarfile.open(tarball).getnames()}
    paths = [makepath(n) for n in names]
    return [p for p in paths if p.isfile() or p.isdir()]


def remove(tarballs):
    for tarball in tarballs:
        for path in removable_contents(tarball):
            if path.isdir():
                print 'rm -rf', path
{'fraction_non_alphanumeric': 0.11608175858079445, 'fraction_numerical': 0.002699575780948708, 'mean_word_length': 2.638148667601683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5873326', 'n_tokens_mistral': 867, 'n_tokens_neox': 821, 'n_words': 265}