<commit_before>import numpy as np

from candidate_selection.models.lazy_indexer import LazyIndexer
from input_models.hypergraph.hypergraph_input_model import HypergraphInputModel


class HypergraphPreprocessor:

    entity_indexer = None
    relation_indexer = None
    in_batch_indices = None
    in_batch_labels = None

    graph_counter = None

    def __init__(self):
        self.entity_indexer = LazyIndexer()
        self.relation_indexer = LazyIndexer()
        self.in_batch_indices = {}
        self.in_batch_labels = {}
        self.graph_counter = 0

    def preprocess(self, hypergraph_batch):
        print("Beginning preprocessing...")
        self.in_batch_indices = {}
        self.in_batch_labels = {}
        self.graph_counter = 0
        vertex_list_slices = np.empty((len(hypergraph_batch),2), dtype=np.int32)

        event_to_entity_edges = np.empty((0,2), dtype=np.int32)
        entity_to_event_edges = np.empty((0,2), dtype=np.int32)
        entity_to_entity_edges = np.empty((0,2), dtype=np.int32)
        event_to_entity_types = np.empty((0,), dtype=np.int32)
        entity_to_event_types = np.empty((0,), dtype=np.int32)
        entity_to_entity_types = np.empty((0,), dtype=np.int32)

        entity_map = np.empty(0, dtype=np.int32)

        event_start_index = 0
        entity_start_index = 0
        for i,hypergraph in enumerate(hypergraph_batch):
            print("Element started")
            phg = self.preprocess_single_hypergraph(hypergraph, event_start_index, entity_start_index)
            vertex_list_slices[i][0] = phg[0]
            vertex_list_slices[i][1] = phg[1]

            event_start_index += phg[0]
            entity_start_index += phg[1]

            if phg[2].shape[0] > 0:
                event_to_entity_edges = np.concatenate((event_to_entity_edges, phg[2]))
                event_to_entity_types = np.concatenate((event_to_entity_types, phg[3]))

            if phg[4].shape[0] > 0:
                entity_to_event_edges = np.concatenate((entity_to_event_edges, phg[4]))
                entity_to_event_types = np.concatenate((entity_to_event_types, phg[5]))

            if phg[6].shape[0] > 0:
                entity_to_entity_edges = np.concatenate((entity_to_entity_edges, phg[6]))
                entity_to_entity_types = np.concatenate((entity_to_entity_types, phg[7]))

            entity_map = np.concatenate((entity_map, phg[8]))

        entity_vertex_slices = vertex_list_slices[:,1]
        print("Getting vertex lookup matrix")
        entity_vertex_matrix = self.get_padded_vertex_lookup_matrix(entity_vertex_slices, hypergraph_batch)

        print(entity_vertex_matrix.shape)

        n_entities = np.max(entity_vertex_matrix)
        n_events = np.sum(vertex_list_slices[:,0])

        input_model = HypergraphInputModel()
        input_model.entity_vertex_matrix = entity_vertex_matrix
        input_model.entity_vertex_slices = entity_vertex_slices
        input_model.entity_map = entity_map
        input_model.event_to_entity_edges = event_to_entity_edges
        input_model.event_to_entity_types = event_to_entity_types
        input_model.entity_to_event_edges = entity_to_event_edges
        input_model.entity_to_event_types = entity_to_event_types
        input_model.entity_to_entity_edges = entity_to_entity_edges
        input_model.entity_to_entity_types = entity_to_entity_types<commit_msg>Prints, prints print<commit_after>import numpy as np

from candidate_selection.models.lazy_indexer import LazyIndexer
from input_models.hypergraph.hypergraph_input_model import HypergraphInputModel


class HypergraphPreprocessor:

    entity_indexer = None
    relation_indexer = None
    in_batch_indices = None
    in_batch_labels = None

    graph_counter = None

    def __init__(self):
        self.entity_indexer = LazyIndexer()
        self.relation_indexer = LazyIndexer()
        self.in_batch_indices = {}
        self.in_batch_labels = {}
        self.graph_counter = 0

    def preprocess(self, hypergraph_batch):
        print("Beginning preprocessing...")
        self.in_batch_indices = {}
        self.in_batch_labels = {}
        self.graph_counter = 0
        vertex_list_slices = np.empty((len(hypergraph_batch),2), dtype=np.int32)

        event_to_entity_edges = np.empty((0,2), dtype=np.int32)
        entity_to_event_edges = np.empty((0,2), dtype=np.int32)
        entity_to_entity_edges = np.empty((0,2), dtype=np.int32)
        event_to_entity_types = np.empty((0,), dtype=np.int32)
        entity_to_event_types = np.empty((0,), dtype=np.int32)
        entity_to_entity_types = np.empty((0,), dtype=np.int32)

        entity_map = np.empty(0, dtype=np.int32)

        event_start_index = 0
        entity_start_index = 0
        for i,hypergraph in enumerate(hypergraph_batch):
            print("Element started")
            phg = self.preprocess_single_hypergraph(hypergraph, event_start_index, entity_start_index)
            vertex_list_slices[i][0] = phg[0]
            vertex_list_slices[i][1] = phg[1]

            event_start_index += phg[0]
            entity_start_index += phg[1]

            if phg[2].shape[0] > 0:
                event_to_entity_edges = np.concatenate((event_to_entity_edges, phg[2]))
                event_to_entity_types = np.concatenate((event_to_entity_types, phg[3]))

            if phg[4].shape[0] > 0:
                entity_to_event_edges = np.concatenate((entity_to_event_edges, phg[4]))
                entity_to_event_types = np.concatenate((entity_to_event_types, phg[5]))

            if phg[6].shape[0] > 0:
                entity_to_entity_edges = np.concatenate((entity_to_entity_edges, phg[6]))
                entity_to_entity_types = np.concatenate((entity_to_entity_types, phg[7]))

            entity_map = np.concatenate((entity_map, phg[8]))

        entity_vertex_slices = vertex_list_slices[:,1]
        print("Getting vertex lookup matrix")
        entity_vertex_matrix = self.get_padded_vertex_lookup_matrix(entity_vertex_slices, hypergraph_batch)

        print(entity_vertex_slices)
        print(entity_vertex_matrix.shape)

        n_entities = np.max(entity_vertex_matrix)
        n_events = np.sum(vertex_list_slices[:,0])

        input_model = HypergraphInputModel()
        input_model.entity_vertex_matrix = entity_vertex_matrix
        input_model.entity_vertex_slices = entity_vertex_slices
        input_model.entity_map = entity_map
        input_model.event_to_entity_edges = event_to_entity_edges
        input_model.event_to_entity_types = event_to_entity_types
        input_model.entity_to_event_edges = entity_to_event_edges
        input_model.entity_to_event_types = entity_to_event_types
        input_model.entity_to_entity_edges = entity_to_entity_edges
        input_model.entity_to_entity_types = entity_to_entity_types
{'fraction_non_alphanumeric': 0.09016273273713532, 'fraction_numerical': 0.016126667644040464, 'mean_word_length': 3.1597560975609755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '670064', 'n_tokens_mistral': 2287, 'n_tokens_neox': 2206, 'n_words': 308}