<commit_before># <EMAIL>
import abc
import itertools
import collections
import re

import hetio.abbreviation

direction_to_inverse = {'forward': 'backward',
                         'backward': 'forward',
                         'both': 'both'}

direction_to_abbrev = {'forward': '>', 'backward': '<', 'both': '-'}
direction_to_unicode_abbrev = {'forward': '→', 'backward': '←', 'both': '–'}

class ElemMask(object):

    def __init__(self):
        self.masked = False

    def is_masked(self):
        return self.masked

    def mask(self):
        self.masked = True

    def unmask(self):
        self.masked = False

class IterMask(object):

    def is_masked(self):
        return any(elem.is_masked() for elem in self.mask_elem_iter())

class BaseGraph(object):

    def __init__(self):
        self.node_dict = dict()
        self.edge_dict = dict()
        self.path_dict = dict()

    def get_node(self, kind):
        return self.node_dict[kind]

    def get_edge(self, edge_tuple):
        return self.edge_dict[edge_tuple]

    def get_nodes(self):
        return iter(self.node_dict.values())

    def get_edges(self, exclude_inverts=True):
        for edge in self.edge_dict.values():
            if exclude_inverts and edge.inverted:
                continue
            yield edge

    def __iter__(self):
        return iter(self.node_dict.values())

    def __contains__(self, item):
        return item in self.node_dict

class BaseNode(ElemMask):

    def __init__(self, identifier):
        ElemMask.__init__(self)
        self.identifier = identifier

    @abc.abstractmethod
    def get_id(self):
        pass

    def __hash__(self):
        try:
            return self.hash_
        except AttributeError:
            return hash(self.get_id())

    def __lt__(self, other):
        return self.get_id() < other.get_id()

    def __eq__(self, other):
        return self.get_id() == other.get_id()

class BaseEdge(ElemMask):

    def __init__(self, source, target):
        ElemMask.__init__(self)
        self.source = source
        self.target = target

    def __hash__(self):
        try:
            return self.hash_
        except AttributeError:
            return hash(self.get_id())

    def __str__(self):
        source, target, kind, direction = self.get_id()
        dir_abbrev = direction_to_abbrev[direction]
        return '{0} {3} {2} {3} {1}'.format(source, target, kind, dir_abbrev)

    def get_unicode_str(self):
        source, target, kind, direction = self.get_id()
        dir_abbrev = direction_to_unicode_abbrev[direction]
        return '{0}{3}{2}{3}{1}'.format(source, target, kind, dir_abbrev)

class BasePath(IterMask):

    def __init__(self, edges):
        assert isinstance(edges, tuple)
        self.edges = edges

    def source(self):
        return self[0].source

    def target(self):
        return self[-1].target

    def get_nodes(self):
        nodes = tuple(edge.source for edge in self)
        nodes = nodes + (self.target(), )
        return nodes

    def inverse_edges(self):
        return tuple(reversed(list(edge.inverse for edge in self)))

    def mask_elem_iter(self):
        for edge in self:
            yield edge
            yield edge.source
        yield self.target()

    def max_overlap(self, others):
        for other in others:
            len_other = len(other)
            if len_other > len(self):
                continue
            if self[:len_other] == other:
                return other
        return None

    def __iter__(self):
        return iter(self.edges)

    def __getitem__(self, key):
        return self.edges[key]

    def __len__(self):
        return len(self.edges)

    def __hash__(self):
        return hash(self.edges)

    def __eq__(self, other):
        return self.edges == other.edges
<commit_msg>Check type for edge/path equality<commit_after># <EMAIL>
import abc
import itertools
import collections
import re

import hetio.abbreviation

direction_to_inverse = {'forward': 'backward',
                         'backward': 'forward',
                         'both': 'both'}

direction_to_abbrev = {'forward': '>', 'backward': '<', 'both': '-'}
direction_to_unicode_abbrev = {'forward': '→', 'backward': '←', 'both': '–'}

class ElemMask(object):

    def __init__(self):
        self.masked = False

    def is_masked(self):
        return self.masked

    def mask(self):
        self.masked = True

    def unmask(self):
        self.masked = False

class IterMask(object):

    def is_masked(self):
        return any(elem.is_masked() for elem in self.mask_elem_iter())

class BaseGraph(object):

    def __init__(self):
        self.node_dict = dict()
        self.edge_dict = dict()
        self.path_dict = dict()

    def get_node(self, kind):
        return self.node_dict[kind]

    def get_edge(self, edge_tuple):
        return self.edge_dict[edge_tuple]

    def get_nodes(self):
        return iter(self.node_dict.values())

    def get_edges(self, exclude_inverts=True):
        for edge in self.edge_dict.values():
            if exclude_inverts and edge.inverted:
                continue
            yield edge

    def __iter__(self):
        return iter(self.node_dict.values())

    def __contains__(self, item):
        return item in self.node_dict

class BaseNode(ElemMask):

    def __init__(self, identifier):
        ElemMask.__init__(self)
        self.identifier = identifier

    @abc.abstractmethod
    def get_id(self):
        pass

    def __hash__(self):
        try:
            return self.hash_
        except AttributeError:
            return hash(self.get_id())

    def __lt__(self, other):
        return self.get_id() < other.get_id()

    def __eq__(self, other):
        return (type(other) is type(self)) and (self.get_id() == other.get_id())

class BaseEdge(ElemMask):

    def __init__(self, source, target):
        ElemMask.__init__(self)
        self.source = source
        self.target = target

    def __hash__(self):
        try:
            return self.hash_
        except AttributeError:
            return hash(self.get_id())

    def __str__(self):
        source, target, kind, direction = self.get_id()
        dir_abbrev = direction_to_abbrev[direction]
        return '{0} {3} {2} {3} {1}'.format(source, target, kind, dir_abbrev)

    def get_unicode_str(self):
        source, target, kind, direction = self.get_id()
        dir_abbrev = direction_to_unicode_abbrev[direction]
        return '{0}{3}{2}{3}{1}'.format(source, target, kind, dir_abbrev)

class BasePath(IterMask):

    def __init__(self, edges):
        assert isinstance(edges, tuple)
        self.edges = edges

    def source(self):
        return self[0].source

    def target(self):
        return self[-1].target

    def get_nodes(self):
        nodes = tuple(edge.source for edge in self)
        nodes = nodes + (self.target(), )
        return nodes

    def inverse_edges(self):
        return tuple(reversed(list(edge.inverse for edge in self)))

    def mask_elem_iter(self):
        for edge in self:
            yield edge
            yield edge.source
        yield self.target()

    def max_overlap(self, others):
        for other in others:
            len_other = len(other)
            if len_other > len(self):
                continue
            if self[:len_other] == other:
                return other
        return None

    def __iter__(self):
        return iter(self.edges)

    def __getitem__(self, key):
        return self.edges[key]

    def __len__(self):
        return len(self.edges)

    def __hash__(self):
        return hash(self.edges)

    def __eq__(self, other):
        return (type(other) is type(self)) and (self.edges == other.edges)

{'fraction_non_alphanumeric': 0.11149780871358597, 'fraction_numerical': 0.0030935808197989174, 'mean_word_length': 2.296091758708581, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047333', 'n_tokens_mistral': 2400, 'n_tokens_neox': 2297, 'n_words': 584}