<commit_before>from __future__ import print_function
import bisect
import abc

__author__ = 'greg'

def index(a, x):
    'Locate the leftmost value exactly equal to x'
    i = bisect.bisect_left(a, x)
    if i != len(a) and a[i] == x:
        return i
    raise ValueError


# code from
# http://stackoverflow.com/questions/2130016/splitting-a-list-of-arbitrary-size-into-only-roughly-n-equal-parts
def chunk_it(seq, num):
    avg = len(seq) / float(num)
    out = []
    last = 0.0

    while last < len(seq):
        out.append(dict(seq[int(last):int(last + avg)]))
        last += avg

    return out


# mappings are for use in dimension reduction
def identity_mapping(markings):
    print("identiy mapping")
    return markings


class Cluster:
    __metaclass__ = abc.ABCMeta

    def __init__(self,shape,project,additional_params):
        """
        :param project_api: how to talk to whatever project we are clustering for (Panoptes/Ouroboros shouldn't matter)
        :param min_cluster_size: minimum number of points in a cluster to not be considered noise
        :return:
        """
        self.shape = shape

        if "reduction" in additional_params:
            self.dim_reduction_alg = additional_params["reduction"]
        else:
            self.dim_reduction_alg = identity_mapping

        self.stats = {}
        self.project = project


    @abc.abstractmethod
    def __cluster__(self,markings,user_ids,tools,reduced_markings,dimensions,subject_id):
        """
        the main function for clustering
        :param user_ids:
        :param markings:
        :param jpeg_file:
        :return cluster_centers: the center of each cluster - probably just take the average along each dimension
        feel free to try something else but the results might not mean as much
        :return markings_per_cluster: the markings in each cluster
        :return users_per_cluster: the user id of each marking per cluster
        :return time_to_cluster: how long it took to cluster
        """
        cluster_centers = []
        markings_per_cluster = []
        users_per_cluster = []
        time_to_cluster = 0

        return (cluster_centers , markings_per_cluster, users_per_cluster), time_to_cluster

    def __aggregate__(self,raw_markings,image_dimensions):
        """
        the function to call from outside to do the clustering
        override but call if you want to add additional functionality
        :param subject_id: what is the subject (in Ouroboros == zooniverse_id)
        :param jpeg_file: for debugging - to show step by step what is happening
        :return:
        """
        aggregation = {}
        # start by calling the api to get the annotations along with the list of who made each marking
        # so for this function, we know that annotations = markings
        # all_markings =  self.project_api.__get_markings__(subject_id,gold_standard)
        # print all_markings
        # self.clusterResults[subject_id] = {"param":"task_id"}
        print("clustering " + self.shape)
        for task_id in raw_markings:
            # go through each marking tool - multiple tools may use the desired shape tool
            # so pay attention to only those markings
            for shape in raw_markings[task_id].keys():
                if shape != self.shape:
                    continue

                for subject_count,subject_id in list(enumerate(raw_markings[task_id][shape])):
                    print(subject_count,subject_id)
                    assert raw_markings[task_id][shape][subject_id] != []

                    # remove any "markings" which correspond to the user not making a marking
                    # these are still useful for noting that the user saw that image
                    pruned_markings = [(u,m,t) for u,m,t in raw_markings[task_id][shape][subject_id] if m is not None]
                    all_users,t1,t2 = zip(*raw_markings[task_id][shape][subject_id])<commit_msg>getting rid of unused print statements<commit_after>from __future__ import print_function
import bisect
import abc

__author__ = 'greg'

def index(a, x):
    'Locate the leftmost value exactly equal to x'
    i = bisect.bisect_left(a, x)
    if i != len(a) and a[i] == x:
        return i
    raise ValueError


# code from
# http://stackoverflow.com/questions/2130016/splitting-a-list-of-arbitrary-size-into-only-roughly-n-equal-parts
def chunk_it(seq, num):
    avg = len(seq) / float(num)
    out = []
    last = 0.0

    while last < len(seq):
        out.append(dict(seq[int(last):int(last + avg)]))
        last += avg

    return out


# mappings are for use in dimension reduction
def identity_mapping(markings):
    return markings


class Cluster:
    __metaclass__ = abc.ABCMeta

    def __init__(self,shape,project,additional_params):
        """
        :param project_api: how to talk to whatever project we are clustering for (Panoptes/Ouroboros shouldn't matter)
        :param min_cluster_size: minimum number of points in a cluster to not be considered noise
        :return:
        """
        self.shape = shape

        if "reduction" in additional_params:
            self.dim_reduction_alg = additional_params["reduction"]
        else:
            self.dim_reduction_alg = identity_mapping

        self.stats = {}
        self.project = project


    @abc.abstractmethod
    def __cluster__(self,markings,user_ids,tools,reduced_markings,dimensions,subject_id):
        """
        the main function for clustering
        :param user_ids:
        :param markings:
        :param jpeg_file:
        :return cluster_centers: the center of each cluster - probably just take the average along each dimension
        feel free to try something else but the results might not mean as much
        :return markings_per_cluster: the markings in each cluster
        :return users_per_cluster: the user id of each marking per cluster
        :return time_to_cluster: how long it took to cluster
        """
        cluster_centers = []
        markings_per_cluster = []
        users_per_cluster = []
        time_to_cluster = 0

        return (cluster_centers , markings_per_cluster, users_per_cluster), time_to_cluster

    def __aggregate__(self,raw_markings,image_dimensions):
        """
        the function to call from outside to do the clustering
        override but call if you want to add additional functionality
        :param subject_id: what is the subject (in Ouroboros == zooniverse_id)
        :param jpeg_file: for debugging - to show step by step what is happening
        :return:
        """
        aggregation = {}
        # start by calling the api to get the annotations along with the list of who made each marking
        # so for this function, we know that annotations = markings
        # all_markings =  self.project_api.__get_markings__(subject_id,gold_standard)
        # self.clusterResults[subject_id] = {"param":"task_id"}
        for task_id in raw_markings:
            # go through each marking tool - multiple tools may use the desired shape tool
            # so pay attention to only those markings
            for shape in raw_markings[task_id].keys():
                if shape != self.shape:
                    continue

                for subject_count,subject_id in list(enumerate(raw_markings[task_id][shape])):
                    assert raw_markings[task_id][shape][subject_id] != []

                    # remove any "markings" which correspond to the user not making a marking
                    # these are still useful for noting that the user saw that image
                    pruned_markings = [(u,m,t) for u,m,t in raw_markings[task_id][shape][subject_id] if m is not None]
                    all_users,t1,t2 = zip(*raw_markings[task_id][shape][subject_id])
{'fraction_non_alphanumeric': 0.07586825579905165, 'fraction_numerical': 0.0030757400999615533, 'mean_word_length': 2.56184390689183, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5175829', 'n_tokens_mistral': 2241, 'n_tokens_neox': 2117, 'n_words': 701}