<commit_before>import collections
import itertools
import logging

import baker
import numpy as np

import yapga.util


log = logging.getLogger('yapga')


@baker.command
def list_changes(filename):
    """List all of the change-ids in `filename`.
    """
    for c in yapga.util.load_changes(filename):
        print(c.data)


@baker.command
def rev_size_vs_count(filename, outfile=None):
    """Log-x scatter of patch size vs. # of commits
    to a review.
    """
    data = []

    for c in yapga.util.load_changes(filename):
        revs = list(c.revisions)
        if revs:
            data.append([len(revs), revs[0].size()])

    data = list(zip(*data))

    import numpy
    print('corr. coeff:', numpy.corrcoef(data))

    import matplotlib.pyplot as plt
    plt.scatter(data[1], data[0], marker=',')
    plt.xscale('log')
    plt.xlim(1, max(data[1]))

    if outfile:
        plt.savefig(outfile)

    plt.show()


@baker.command
def rev_count_hist(filename, outfile=None):
    "Histogram of number of revision per change."
    changes = list(yapga.util.load_changes(filename))

    log.info('Scanning {} changes'.format(len(changes)))

    vals = [len(list(c.revisions)) for c in changes]

    import matplotlib.pyplot as plt
    plt.hist(vals, bins=30)

    if outfile:
        plt.savefig(outfile)

    plt.show()


@baker.command
def changes_by_owner(filename):
    "Simple histogram of change count by owners."
    counts = {}

    for change in yapga.util.load_changes(filename):
        o = change.owner.name
        counts[o] = counts.get(o, 0) + 1

    counts = sorted(counts.items(), key=lambda x: x[1])
    for c in counts:
        print((c[1] // 10) * '*', c[0])


@baker.command
def list_messages(filename):
    "List all of the change messages in a changes file."
    for change in yapga.util.load_changes(filename):
        for msg in change.messages:
            print(msg)

@baker.command
def list_reviewers(filename):
    "List all of the reviewers in a reviews file."
    for change in yapga.util.load_changes(filename):
        for r in change.reviewers:
            print(r.name)


@baker.command
def compare_reviewers(changes, reviews):
    """Heatmap showing how often reviewers review change owners.

    Not very useful right now. Need to weed out the useless,
    one-offers and stuff.
    """
    import math
    import matplotlib.pyplot as plt

    # http://stackoverflow.com/questions/14391959/heatmap-in-matplotlib-with-pcolor
    reviews = dict(yapga.util.load_reviews(reviews))
    changes = list(yapga.util.load_changes(changes))

    owners = list(sorted(set(
        c.owner.name for c in changes)))
    reviewers = list(sorted(set(
        r.name for revs in reviews.values() for r in revs)))

    if not owners:
        print('Owners list is empty. Aborting.')
        return

    if not reviewers:
        print('Reviewers list is empty. Aborting.')
        return

    # This is a matrix of reviewer to owner, where each cell is a
    # count of how many times a reviewer reviewed a particular owner
    data = np.zeros((len(reviewers), len(owners)))

    for change in changes:
        try:
            owner_idx = yapga.util.index_of(owners, change.owner.name)<commit_msg>Added "low order" trimming of data in heatmap.<commit_after>import collections
import itertools
import logging

import baker
import numpy as np

import yapga.util


log = logging.getLogger('yapga')


@baker.command
def list_changes(filename):
    """List all of the change-ids in `filename`.
    """
    for c in yapga.util.load_changes(filename):
        print(c.data)


@baker.command
def rev_size_vs_count(filename, outfile=None):
    """Log-x scatter of patch size vs. # of commits
    to a review.
    """
    data = []

    for c in yapga.util.load_changes(filename):
        revs = list(c.revisions)
        if revs:
            data.append([len(revs), revs[0].size()])

    data = list(zip(*data))

    import numpy
    print('corr. coeff:', numpy.corrcoef(data))

    import matplotlib.pyplot as plt
    plt.scatter(data[1], data[0], marker=',')
    plt.xscale('log')
    plt.xlim(1, max(data[1]))

    if outfile:
        plt.savefig(outfile)

    plt.show()


@baker.command
def rev_count_hist(filename, outfile=None):
    "Histogram of number of revision per change."
    changes = list(yapga.util.load_changes(filename))

    log.info('Scanning {} changes'.format(len(changes)))

    vals = [len(list(c.revisions)) for c in changes]

    import matplotlib.pyplot as plt
    plt.hist(vals, bins=30)

    if outfile:
        plt.savefig(outfile)

    plt.show()


@baker.command
def changes_by_owner(filename):
    "Simple histogram of change count by owners."
    counts = {}

    for change in yapga.util.load_changes(filename):
        o = change.owner.name
        counts[o] = counts.get(o, 0) + 1

    counts = sorted(counts.items(), key=lambda x: x[1])
    for c in counts:
        print((c[1] // 10) * '*', c[0])


@baker.command
def list_messages(filename):
    "List all of the change messages in a changes file."
    for change in yapga.util.load_changes(filename):
        for msg in change.messages:
            print(msg)

@baker.command
def list_reviewers(filename):
    "List all of the reviewers in a reviews file."
    for change in yapga.util.load_changes(filename):
        for r in change.reviewers:
            print(r.name)


@baker.command
def compare_reviewers(changes, reviews, filter_rate=0.0):
    """Heatmap showing how often reviewers review change owners.

    Not very useful right now. Need to weed out the useless,
    one-offers and stuff.
    """
    import math
    import matplotlib.pyplot as plt

    # http://stackoverflow.com/questions/14391959/heatmap-in-matplotlib-with-pcolor
    reviews = dict(yapga.util.load_reviews(reviews))
    changes = list(yapga.util.load_changes(changes))

    owners = collections.defaultdict(lambda: 0)
    for change in changes:
        owners[change.owner.name] += 1

    owners = list(sorted(
        # Extract just the names from ...
        x[0] for x in

        # ...the first N (i.e. the people with the most
        # changes) of...
        itertools.islice(

            # ...owners dict sorted by number of changes.
            sorted(owners.items(),
                   key=lambda x: x[1],
                   reverse=True),
            math.ceil((1 - filter_rate) * len(owners)))))

    reviewers = collections.defaultdict(lambda: 0)
    for revs in reviews.values():
        for r in revs:
            reviewers[r.name] += 1

    reviewers = list(sorted(
        x[0] for x in
        itertools.islice(
            sorted(reviewers.items(),
                   key=lambda x: x[1],
                   reverse=True),
            math.ceil((1 - filter_rate) * len(reviewers)))))

    #owners = list(sorted(set(
    #    c.owner.name for c in changes)))
    #reviewers = list(sorted(set(
    #    r.name for revs in reviews.values() for r in revs)))

    if not owners:
        print('Owners list is empty. Aborting.')
        return

    if not reviewers:
        print('Reviewers list is empty. Aborting.')
        return

    # This is a matrix of reviewer to owner, where each cell is a
    # count of how many times a reviewer reviewed a particular owner
    data = np.zeros((len(reviewers), len(owners)))

    for change in changes:
        try:
            owner_idx = yapga.util.index_of(owners, change.owner.name)
{'fraction_non_alphanumeric': 0.11513113651647612, 'fraction_numerical': 0.007531943510423672, 'mean_word_length': 2.6202531645569622, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1631636', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2351, 'n_words': 668}