<commit_before>from collections import defaultdict


class EdgeFilter:

    edge_counts = None
    relation_indexer = None

    def __init__(self, inner, edge_list_file, relation_indexer=None):
        self.inner = inner
        self.edge_counts = defaultdict(int)
        self.load_edge_list(edge_list_file)
        self.relation_indexer = None

    def load_edge_list(self, file):
        for line in open(file):
            parts = line.strip().split(' ')

            edge_name = parts[1]
            edge_count = int(parts[0])

            if self.relation_indexer is not None:
                edge_name = self.relation_indexer.index(edge_name)

            self.edge_counts[edge_name] = int(edge_count)

EdgeFilter(None, "/home/michael/Projects/QuestionAnswering/GCNQA/data/webquestions/edge_count.txt")<commit_msg>frequency filter<commit_after>from collections import defaultdict
import numpy as np


class EdgeFilter:

    edge_counts = None
    relation_indexer = None

    def __init__(self, inner, edge_list_file, relation_indexer=None):
        self.inner = inner
        self.edge_counts = defaultdict(int)
        self.load_edge_list(edge_list_file)
        self.relation_indexer = relation_indexer

    def load_edge_list(self, file):
        for line in open(file):
            parts = line.strip().split(' ')

            edge_name = parts[1]
            edge_count = int(parts[0])

            if self.relation_indexer is not None:
                edge_name = self.relation_indexer.index(edge_name)

            self.edge_counts[edge_name] = int(edge_count)

    def enrich(self, instance):
        self.inner.enrich(instance)
        edges = instance["neighborhood"].entity_to_event_edges
        filtered_edges = self.filter_edges(edges)
        instance["neighborhood"].entity_to_event_edges = filtered_edges

        edges = instance["neighborhood"].entity_to_entity_edges
        filtered_edges = self.filter_edges(edges)
        instance["neighborhood"].entity_to_entity_edges = filtered_edges

        edges = instance["neighborhood"].event_to_entity_edges
        filtered_edges = self.filter_edges(edges)
        instance["neighborhood"].event_to_entity_edges = filtered_edges

    def filter_edges(self, edges):
        counts = np.array([self.edge_counts[e] for e in edges[:, 1]])
        filtered_edges = edges[np.where(counts > self.edge_counts)]
        return filtered_edges


EdgeFilter(None, "/home/michael/Projects/QuestionAnswering/GCNQA/data/webquestions/edge_count.txt")
{'fraction_non_alphanumeric': 0.09206855320844959, 'fraction_numerical': 0.0019928258270227183, 'mean_word_length': 3.0549273021001615, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1117380', 'n_tokens_mistral': 766, 'n_tokens_neox': 735, 'n_words': 133}