<commit_before>import git
import util

class Search(object):
    header_titles = ["author", "author-mail", "author-time", "author-tz",
                    "committer", "committer-mail", "committer-time", "committer-tz",
                    "summary", "previous", "filename", "\t"]
    def __init__(self, repo_path):
        self.repo_path = repo_path
        self.repo = git.Repo(repo_path, odbt=git.GitDB)

        self.authors = []

    def find_author(self, name=None, email=None, add_author=False):
        """
        Find author with given name and/or email. If author does not exist,
        add the author if add_author is True.
        """

        for author in self.authors:
            if name and email and author.name == name and author.email == email:
                return author
            elif name and author.name == name:
                return author
            elif author.name == email:
                return author
        if add_author:
            person = Person(name, email)
            self.authors.append(person)
            return person
        return None

    def score_last_commit(self, block):
        scores = {}
        contributors, num_lines_total = self.lines_contributed(block)
        for contributor, contribution in contributors.items():
            scores[contributor] = float(contribution)/num_lines_total
        return scores


    def lines_contributed(self, block, rev="HEAD"):
        """
        Given a block, return a hash {author: num lines contributed}.
        """

        contributions = {}      # author => num lines
        shas = set()

        blamestr = self.repo.git.blame(rev, block, incremental=True)
        lines = blamestr.splitlines()

        num_lines_total = 0

        sha = None
        i = 0
        while i < len(lines):
            # We are at the first line of group or sub-group.
            sha, ln_orig, ln_final, ln_group = lines[i].split()
            ln_group = int(ln_group)

            num_lines_total += ln_group

            if not shas.issuperset((sha,)):
                shas.add(sha)

                # We are at a new commit group. Figure out the author.
                author_name, author_email = None, None
                while not (author_name and author_email):
                    i += 1<commit_msg>Set up for a lines_contributed refactoring<commit_after>import git
import util

class Search(object):
    header_titles = ["author", "author-mail", "author-time", "author-tz",
                    "committer", "committer-mail", "committer-time", "committer-tz",
                    "summary", "previous", "filename", "\t"]
    def __init__(self, repo_path):
        self.repo_path = repo_path
        self.repo = git.Repo(repo_path, odbt=git.GitDB)

        self.authors = []

    def rev_list(self, block, rev='HEAD'):
        """
        Return list of revision hashes. Ordered from earliest to latest.
        """
        revs = self.repo.rev_list(rev, block).split()
        revs.reverse()  # earliest commits first
        return revs

    def lines_contributed_for_revs(self, block, revs):
        contributions = []  # [(Person, num lines contributed)]
        for rev in revs:
            rev_contributions = self.lines_contributed(block, rev)


    def find_author(self, name=None, email=None, add_author=False):
        """
        Find author with given name and/or email. If author does not exist,
        add the author if add_author is True.
        """

        for author in self.authors:
            if name and email and author.name == name and author.email == email:
                return author
            elif name and author.name == name:
                return author
            elif author.name == email:
                return author
        if add_author:
            person = Person(name, email)
            self.authors.append(person)
            return person
        return None

    def score_last_commit(self, block):
        """
        Implementation of Score_last_commit(author, block).

        Returns a hash of author to the contribution [0, 1] of the author
        for this particular block. Uses the current statistics of the block.
        """
        scores = {}
        contributors, num_lines_total = self.lines_contributed(block)
        for contributor, contribution in contributors.items():
            scores[contributor] = float(contribution)/num_lines_total
        return scores


    def lines_contributed(self, block, rev="HEAD"):
        """
        Given a block, return a dict {author: num lines contributed}.
        """

        contributions = {}      # author => num lines
                                # {sha => (Person, num lines)}
        #contributions = []      # [(Person, num lines, hash)]
        shas = set()

        blamestr = self.repo.git.blame(rev, block, incremental=True)
        lines = blamestr.splitlines()

        num_lines_total = 0

        sha = None
        i = 0
        while i < len(lines):
            # We are at the first line of group or sub-group.
            sha, ln_orig, ln_final, ln_group = lines[i].split()
            ln_group = int(ln_group)

            num_lines_total += ln_group

            if not shas.issuperset((sha,)):
                shas.add(sha)

                # We are at a new commit group. Figure out the author.
                author_name, author_email = None, None
                while not (author_name and author_email):
                    i += 1
{'fraction_non_alphanumeric': 0.09653284671532847, 'fraction_numerical': 0.00145985401459854, 'mean_word_length': 1.8726415094339623, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '849176', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1459, 'n_words': 452}