<commit_before>from git import *
import git
import pandas as pd
import numpy as np
from time import mktime
from datetime import datetime
from .entity_resolution import entity_resolve
import networkx as nx
from config.config import CONFIG
from . import utils

ALL_ATTRIBUTES = CONFIG.all_attributes #["HEXSHA", "Committer Name", "Committer Email", "Commit Message", "Time", "Parent Commit", "Touched File"]

def cache_fixer(r): 
    """Adds info from row to graph."""
    r["Touched File"] = [x.strip() for x in r["Touched File"][1:-1].split(",")]
    r["Time"] = pd.to_datetime(r["Time"]);
    return r


class GitRepo(object):
    """
    Store a git repository given the address to that repo relative to this file.
    
    It returns the data in many forms.
    """

    def __init__(self, name, url=None, attribs = ALL_ATTRIBUTES, cache=None):
        """
        Index a Pandas DataFrame object by time.
        
        That stores the raw form of the repo's commit data as a table.
         
        Each row in this table  is a commit.
        
        And each column represents an attribute of that commit:
        (eg.: time, message, commiter name, committer email, commit hexsha).
        """

        self._commit_data = None;
        self.url = url;
        self.repo = None
        self.name = name;
        
        if cache is None:
            self.repo = Repo(url)
            self.populate_data(ALL_ATTRIBUTES)
        else:
            cache = cache.apply(cache_fixer, axis=1)
            cache.set_index(cache["Time"])
            self._commit_data = cache;

            missing = list();
            cols = self.commit_data.columns
            for attr in attribs:
                if attr not in cols and str(attr) not in cols:
                    missing.append(attr);

            if len(missing) > 0:
                print(("There were " + str(len(missing)) + " missing attributes: "))
                print(missing);

        if ("Committer Name" in attribs and "Committer Email" in attribs):
            self._commit_data["Person-ID"] = None;
            self._commit_data = self._commit_data.apply(lambda row: entity_resolve(row, "Committer Email", "Committer Name"), axis=1)

    def gen_data(self, repo, raw):
        """Generate data to repo."""

        if not repo.active_branch.is_valid():
            print(("Found an empty repo: " + str(self.name)))
            return;
        first = repo.commit()
        commit = first
        firstHexSha = first.hexsha;
        generator = git.Commit.iter_items(repo, firstHexSha);
        
        if "Touched File" in raw:
            print("WARNING: Currently going through file diffs. This will take a very long time (1 minute per 3000 commits.) We suggest using a small repository.")
        for commit in generator:
            try: 
                if "Touched File" in raw:
                    diff_list = list();
                    for diff in commit.diff(commit.parents[0]):
                        if diff.b_blob:
                            diff_list.append(diff.b_blob.path);
                        else:
                            diff_list.append(diff.a_blob.path);
                    raw["Touched File"].append(diff_list)

                if "Committer Name" in raw:
                    raw["Committer Name"].append(commit.committer.name)
                
                if "Committer Email" in raw:
                    raw["Committer Email"].append(commit.committer.email)
                
                if "Commit Message" in raw:
                    raw["Commit Message"].append(commit.message)
                
                if "Time" in raw or True: # TODO: For now, we always ask for the time
                    raw["Time"].append(pd.to_datetime(commit.committed_date, unit = "s"));
                
                if "Parent Commit" in raw:
                    raw["Parent Commit"].append([par.hexsha for par in commit.parents])
                
                if "HEXSHA" in raw:
                    raw["HEXSHA"].append(commit.hexsha)
            except LookupError:
                print("failed to add a commit because of an encoding error")


    def populate_data(self, attribs = ALL_ATTRIBUTES):
        """Populate data."""
        raw = dict()
        for attrib in attribs:
            raw[attrib] = list();
        repo = self.repo
        self.gen_data(repo, raw);
        print((type(raw["Time"])))
        
        # TODO: NEEDS TIME
        time_index = pd.DatetimeIndex(raw["Time"], periods = 24)
        time_index = utils.add_freq(time_index, freq = "H")
        self._commit_data = pd.DataFrame(raw, index = time_index);<commit_msg>git_repo.py: changed line 122f as it raised TypeError in tests<commit_after>from git import *
import git
import pandas as pd
import numpy as np
from time import mktime
from datetime import datetime
from .entity_resolution import entity_resolve
import networkx as nx
from config.config import CONFIG
from . import utils

ALL_ATTRIBUTES = CONFIG.all_attributes #["HEXSHA", "Committer Name", "Committer Email", "Commit Message", "Time", "Parent Commit", "Touched File"]

def cache_fixer(r):
    """Adds info from row to graph."""
    r["Touched File"] = [x.strip() for x in r["Touched File"][1:-1].split(",")]
    r["Time"] = pd.to_datetime(r["Time"]);
    return r


class GitRepo(object):
    """
    Store a git repository given the address to that repo relative to this file.

    It returns the data in many forms.
    """

    def __init__(self, name, url=None, attribs = ALL_ATTRIBUTES, cache=None):
        """
        Index a Pandas DataFrame object by time.

        That stores the raw form of the repo's commit data as a table.

        Each row in this table  is a commit.

        And each column represents an attribute of that commit:
        (eg.: time, message, commiter name, committer email, commit hexsha).
        """

        self._commit_data = None;
        self.url = url;
        self.repo = None
        self.name = name;

        if cache is None:
            self.repo = Repo(url)
            self.populate_data(ALL_ATTRIBUTES)
        else:
            cache = cache.apply(cache_fixer, axis=1)
            cache.set_index(cache["Time"])
            self._commit_data = cache;

            missing = list();
            cols = self.commit_data.columns
            for attr in attribs:
                if attr not in cols and str(attr) not in cols:
                    missing.append(attr);

            if len(missing) > 0:
                print(("There were " + str(len(missing)) + " missing attributes: "))
                print(missing);

        if ("Committer Name" in attribs and "Committer Email" in attribs):
            self._commit_data["Person-ID"] = None;
            self._commit_data = self._commit_data.apply(lambda row: entity_resolve(row, "Committer Email", "Committer Name"), axis=1)

    def gen_data(self, repo, raw):
        """Generate data to repo."""

        if not repo.active_branch.is_valid():
            print(("Found an empty repo: " + str(self.name)))
            return;
        first = repo.commit()
        commit = first
        firstHexSha = first.hexsha;
        generator = git.Commit.iter_items(repo, firstHexSha);

        if "Touched File" in raw:
            print("WARNING: Currently going through file diffs. This will take a very long time (1 minute per 3000 commits.) We suggest using a small repository.")
        for commit in generator:
            try:
                if "Touched File" in raw:
                    diff_list = list();
                    for diff in commit.diff(commit.parents[0]):
                        if diff.b_blob:
                            diff_list.append(diff.b_blob.path);
                        else:
                            diff_list.append(diff.a_blob.path);
                    raw["Touched File"].append(diff_list)

                if "Committer Name" in raw:
                    raw["Committer Name"].append(commit.committer.name)

                if "Committer Email" in raw:
                    raw["Committer Email"].append(commit.committer.email)

                if "Commit Message" in raw:
                    raw["Commit Message"].append(commit.message)

                if "Time" in raw or True: # TODO: For now, we always ask for the time
                    raw["Time"].append(pd.to_datetime(commit.committed_date, unit = "s"));

                if "Parent Commit" in raw:
                    raw["Parent Commit"].append([par.hexsha for par in commit.parents])

                if "HEXSHA" in raw:
                    raw["HEXSHA"].append(commit.hexsha)
            except LookupError:
                print("failed to add a commit because of an encoding error")


    def populate_data(self, attribs = ALL_ATTRIBUTES):
        """Populate data."""
        raw = dict()
        for attrib in attribs:
            raw[attrib] = list();
        repo = self.repo
        self.gen_data(repo, raw);
        print((type(raw["Time"])))

        time_index = pd.DatetimeIndex(raw["Time"]).to_period("H")
        time_index = utils.add_freq(time_index, freq=None)
        self._commit_data = pd.DataFrame(raw, index = time_index);
{'fraction_non_alphanumeric': 0.10300662107891023, 'fraction_numerical': 0.002930641484858352, 'mean_word_length': 1.8884012539184953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4366562', 'n_tokens_mistral': 2599, 'n_tokens_neox': 2490, 'n_words': 794}