<commit_before>from global_variables.global_variable import InformationType
import json


class StructuredPrecedent:

    CATEGORY_INDEX = 0
    FACTS = "facts"
    FACTS_VECTOR = "facts_vector"
    OUTCOMES = "outcomes"
    OUTCOMES_VECTOR = "outcomes_vector"
    LEASE_TERMINATED = "lease_terminated"

    def __init__(self, fact_labels, fact_data_tuple, outcome_labels, outcome_data_tuple):
        """
        :param fact_labels: Facts cluster labels
        :param fact_data_tuple ([int], [string], [string]): vectors, transformed sentences, original sentence 
        :param outcome_labels: Outcomes cluster labels
        :param outcome_data_tuple ([int], [string], [string]): vectors, transformed sentences, original sentence  
        """
        self.precedents = {}
        self.create_vector(fact_labels, fact_data_tuple, self.FACTS)
        self.create_vector(outcome_labels, outcome_data_tuple, self.OUTCOMES_VECTOR)

    def create_vector(self, labels, data_tuple, data_type):
        """
        Creates a fact or outcome vector for all precedents
        :param labels (int): Cluster labels
        :param data_tuple ([int], [string], [string]): vectors, transformed sentences, original sentence
        :param data_type (string): "facts" or "outcomes"
        :return: Void
        """
        unique_labels = set(labels)
        for label in (temp_label for temp_label in unique_labels if temp_label >= 0):
            for raw_fact_tuple, file_name_tuple in zip(enumerate(data_tuple[InformationType.FACTS.value][labels == label]),
                                                       enumerate(data_tuple[InformationType.PRECEDENTS_FILE_NAMES.value][labels == label])):

                file_name = file_name_tuple[1].replace(".txt", "")

                if file_name not in self.precedents:

                    # Create new entry with file name
                    self.precedents[file_name] = dict.fromkeys([self.FACTS, self.FACTS_VECTOR, self.OUTCOMES, self.OUTCOMES_VECTOR, self.LEASE_TERMINATED])

                    # initialize fields
                    self.precedents[file_name][self.FACTS_VECTOR] = [0] * len(unique_labels)
                    self.precedents[file_name][self.FACTS] = []
                    self.precedents[file_name][self.OUTCOMES_VECTOR] = [0] * len(unique_labels)
                    self.precedents[file_name][self.OUTCOMES] = []

                # populate fields
                if data_type == self.FACTS:
                    self.precedents[file_name][self.FACTS_VECTOR][raw_fact_tuple[self.CATEGORY_INDEX]] = 1  # 1 signifies that the fact exists
                    self.precedents[file_name][self.FACTS].append(raw_fact_tuple)
                else:
                    self.precedents[file_name][self.OUTCOMES_VECTOR][raw_fact_tuple[self.CATEGORY_INDEX]] = 1  # 1 signifies that the fact exists
                    self.precedents[file_name][self.OUTCOMES].append(raw_fact_tuple)

    def write_data_to_output_dir(self):
        with open("precedent_JSON.txt", 'w') as output_file:
            json.dump(self.precedents, output_file)<commit_msg>[#151] using numpy array instead of list. Can now write to binary<commit_after>import numpy
import joblib
from global_variables.global_variable import InformationType
from global_variables.global_variable import Global

class StructuredPrecedent:

    CATEGORY_INDEX = 0
    FACTS = "facts"
    FACTS_VECTOR = "facts_vector"
    OUTCOMES = "outcomes"
    OUTCOMES_VECTOR = "outcomes_vector"

    def __init__(self, fact_labels, fact_data_tuple, outcome_labels, outcome_data_tuple):
        """
        :param fact_labels: Facts cluster labels
        :param fact_data_tuple ([int], [string], [string]): vectors, transformed sentences, original sentence 
        :param outcome_labels: Outcomes cluster labels
        :param outcome_data_tuple ([int], [string], [string]): vectors, transformed sentences, original sentence  
        """
        self.precedents = {}
        self.create_vector(fact_labels, fact_data_tuple, self.FACTS)
        self.create_vector(outcome_labels, outcome_data_tuple, self.OUTCOMES_VECTOR)
        self.write_data_as_bin()

    def create_vector(self, labels, data_tuple, data_type):
        """
        Creates a fact or outcome vector for all precedents
        :param labels (int): Cluster labels
        :param data_tuple ([int], [string], [string]): vectors, transformed sentences, original sentence
        :param data_type (string): "facts" or "outcomes"
        :return: Void
        """
        unique_labels_size = len(set(labels)) - 1  # size is reduces by one because we need to ignore label -1
        for index, label in enumerate(labels):
            if label < 0:
                continue

            raw_fact = data_tuple[InformationType.FACTS.value][index]

            for file_name in data_tuple[InformationType.PRECEDENTS_FILE_NAMES.value][index]:
                file_name = file_name.replace(".txt", "")
                if file_name not in self.precedents:

                    # Create new entry with file name
                    self.precedents[file_name] = dict.fromkeys([self.FACTS, self.FACTS_VECTOR, self.OUTCOMES, self.OUTCOMES_VECTOR])

                    # initialize fields
                    self.precedents[file_name][self.FACTS_VECTOR] = numpy.zeros(unique_labels_size, dtype=numpy.int)
                    self.precedents[file_name][self.FACTS] = []
                    self.precedents[file_name][self.OUTCOMES_VECTOR] = numpy.zeros(unique_labels_size, dtype=numpy.int)
                    self.precedents[file_name][self.OUTCOMES] = []

                # populate fields
                if data_type == self.FACTS:
                    self.precedents[file_name][self.FACTS_VECTOR][label] = 1  # 1 signifies that the fact exists
                    self.precedents[file_name][self.FACTS].append(raw_fact)
                else:
                    self.precedents[file_name][self.OUTCOMES_VECTOR][label] = 1  # 1 signifies that the fact exists
                    self.precedents[file_name][self.OUTCOMES].append(raw_fact)

    def write_data_as_bin(self):
        joblib.dump(self.precedents, Global.output_directory + "structured_precedent.bin")
{'fraction_non_alphanumeric': 0.09384541218062029, 'fraction_numerical': 0.0032138839787883657, 'mean_word_length': 2.678486997635934, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3626105', 'n_tokens_mistral': 1895, 'n_tokens_neox': 1760, 'n_words': 385}