<commit_before>from git import Repo

class GitDistiller:
    def __init__(self, git_dir):
        pass
        self.org_repo = Repo(git_dir)

    def clone_repo(self, new_repo_path):
        self.new_repo = self.org_repo.clone(new_repo_path)

    def distille(self, new_repo_path, extensions):
        self.clone_repo(new_repo_path)
        git = self.new_repo.git
        cmd = ['tree-filter', ','.join(self.create_remove_command(extensions))]
        git.filter_branch(cmd)

    def create_remove_command(self, extensions):
        assert len(extensions) > 0
        cmd = ['find']
        first_extension = extensions.pop(0)
        cmd.extend(['-name', '*.%s' % (first_extension)])
        for extension in extensions:
            cmd.extend(['-name', '*.%s' % (extension)])
        cmd.extend(['-type', 'd'])
        print cmd
        return cmd
if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Git Distiller')
    parser.add_argument('repo_dir',
            help='path of repository dir')
    parser.add_argument('new_repo',
            help='path of new repository')
    parser.add_argument('extension', nargs='*')
    parser.add_argument('--delete-no-extension')
    args = parser.parse_args()

    distiller = GitDistiller(args.repo_dir)
    distiller.distille(args.new_repo, args.extension)

<commit_msg>distiller一応完成<commit_after>from git import Repo
import os

class GitDistiller:
    def __init__(self, git_dir):
        pass
        self.org_repo = Repo(git_dir)

    def clone_repo(self, new_repo_path):
        self.new_repo = self.org_repo.clone(new_repo_path)

    def distille(self, new_repo_path, extensions):
        new_repo_path = os.path.abspath(new_repo_path)
        self.clone_repo(new_repo_path)
        git = self.new_repo.git
        execcmd = self.create_remove_command_find(extensions)
        #execcmd = self.create_remove_command_xargs(extensions)
        cmd = ['--tree-filter', execcmd]

        git.filter_branch(cmd)

    def prune_empty(self):
        git = self.new_repo.git
        kwargs = {'f':True, 'prune-empty':True}
        git.filter_branch(**kwargs)

    def create_remove_command_find(self, extensions):
        assert len(extensions) > 0
        cmd = ['find', '.', '\\!', '\\(']
        first_extension = extensions.pop(0)
        cmd.extend(['-name', '\'*.%s\'' % (first_extension)])
        for extension in extensions:
            cmd.extend(['-o', '-name', '\'*.%s\'' % (extension)])
        cmd.extend(['-o', '-type', 'd'])
        cmd.append('\\)')
        cmd.extend(['-exec', 'rm', '{}', '\;'])

        return ' '.join(cmd)

    def create_remove_command_xargs(self, extensions):
        assert len(extensions) > 0
        cmd = ['find', '.', '\\!', '\\(']
        first_extension = extensions.pop(0)
        cmd.extend(['-name', '\'*.%s\'' % (first_extension)])
        for extension in extensions:
            cmd.extend(['-o', '-name', '\'*.%s\'' % (extension)])
        cmd.extend(['-o', '-type', 'd'])
        cmd.append('\\)')
        cmd.extend(['-print0', '|', 'xargs', '-0', 'rm'])
        print cmd
        return ' '.join(cmd)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Git Distiller')
    parser.add_argument('repo_dir',
            help='path of repository dir')
    parser.add_argument('new_repo',
            help='path of new repository')
    parser.add_argument('extension', nargs='*')
    parser.add_argument('--delete-no-extension')
    args = parser.parse_args()

    distiller = GitDistiller(args.repo_dir)
    distiller.distille(args.new_repo, args.extension)


{'fraction_non_alphanumeric': 0.15839956152370513, 'fraction_numerical': 0.0021923814743765417, 'mean_word_length': 2.747433264887064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7085732', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1165, 'n_words': 214}