<commit_before>import uuid
import copy
import logging
import pendulum
import datetime

from django.apps import apps
from django.core.exceptions import FieldDoesNotExist

from share.disambiguation import GraphDisambiguator
from share.util import TopographicalSorter
from share.util import IDObfuscator


logger = logging.getLogger(__name__)


class GraphParsingException(Exception):
    pass


class UnresolvableReference(GraphParsingException):
    pass


class GraphEdge:

    # TODO Cache
    @property
    def field(self):
        subject_model = self.subject.model
        related_model = self.related.model._meta.concrete_model
        possible = tuple(
            f for f in subject_model._meta.get_fields()
            if f.is_relation
            and f.related_model is related_model
            and (not self._hint or f.name == self._hint)
        )
        assert len(possible) == 1

        return possible[0]

    @property
    def remote_field(self):
        return self.field.remote_field

    @property
    def name(self):
        return self.field.name

    @property
    def remote_name(self):
        return self.remote_field.name

    def __init__(self, subject, related, hint=None):
        self.subject = subject
        self.related = related
        self._hint = hint

    def __eq__(self, other):
        return type(self) is type(other) and self.related == other.related and self.subject == other.subject

    def __hash__(self):
        return hash((self.subject, self.related))

    def __repr__(self):
        return '<{}({}, {}, {})>'.format(self.__class__.__name__, self.name, self.subject, self.related)


class ChangeGraph:

    def __init__(self, data, disambiguate=True, namespace=None):
        self.nodes = []
        self.relations = {}
        self._lookup = {}
        self.namespace = namespace

        hints, relations = {}, set()
        for blob in copy.deepcopy(data):
            id, type = blob.pop('@id'), blob.pop('@type')

            self._lookup[id, type] = ChangeNode(self, id, type, blob, namespace=namespace)
            self.relations[self._lookup[id, type]] = set()
            self.nodes.append(self._lookup[id, type])

            for k, v in tuple(blob.items()):
                if isinstance(v, dict) and k != 'extra' and not k.startswith('@'):
                    related = (v.pop('@id'), v.pop('@type'))
                    hints[(id, type), related] = k
                    relations.add(((id, type), related))
                    blob.pop(k)
                if isinstance(v, list):
                    for rel in v:
                        subject = (rel.pop('@id'), rel.pop('@type'))
                        relations.add((subject, (id, type)))
                    blob.pop(k)

        for subject, related in relations:
            try:
                edge = GraphEdge(self._lookup[subject], self._lookup[related], hint=hints.get((subject, related)))
            except KeyError as e:
                raise UnresolvableReference(*e.args)

            self.relations[self._lookup[subject]].add(edge)
            self.relations[self._lookup[related]].add(edge)

        # if disambiguate:
        #     self.disambiguate()

        self.nodes = TopographicalSorter(self.nodes, dependencies=lambda n: tuple(e.related for e in n.related(backward=False))).sorted()

    def disambiguate(self):
        GraphDisambiguator().disambiguate(self)
        self.nodes = TopographicalSorter(self.nodes, dependencies=lambda n: tuple(e.related for e in n.related(backward=False))).sorted()

    def normalize(self):
        for node in self.nodes:
            # This feels overly hacky
            if hasattr(node.model, 'normalize'):
                node.model.normalize(node, self)

    def get(self, id, type):
        return self._lookup[(id, type)]

    def create(self, id, type, attrs):
        return self.add(ChangeNode(self, id or '_:{}'.format(uuid.uuid4()), type, attrs, namespace=self.namespace))

    def add(self, node):
        node.graph = self
        self.nodes.append(node)
        self.relations[node] = set()
        self._lookup[node.id, node.type] = node
        return node

    def relate(self, subject, related, hint=None):
        edge = GraphEdge(subject, related, hint)
        self.relations[subject].add(edge)
        self.relations[related].add(edge)
        return edge

    def replace(self, source, replacement):
        for edge in tuple(source.related()):
            # NOTE: Order of add & removes matters here due to
            # the hash function of an edge and how sets work
            self.relations[source].remove(edge)
            if edge.subject == source:
                edge.subject = replacement
            else:
                edge.related = replacement<commit_msg>Don't modify GraphEdges that are in a set<commit_after>import uuid
import copy
import logging
import pendulum
import datetime

from django.apps import apps
from django.core.exceptions import FieldDoesNotExist

from share.disambiguation import GraphDisambiguator
from share.util import TopographicalSorter
from share.util import IDObfuscator


logger = logging.getLogger(__name__)


class GraphParsingException(Exception):
    pass


class UnresolvableReference(GraphParsingException):
    pass


class GraphEdge:

    # TODO Cache
    @property
    def field(self):
        subject_model = self.subject.model
        related_model = self.related.model._meta.concrete_model
        possible = tuple(
            f for f in subject_model._meta.get_fields()
            if f.is_relation
            and f.related_model is related_model
            and (not self._hint or f.name == self._hint)
        )
        assert len(possible) == 1

        return possible[0]

    @property
    def remote_field(self):
        return self.field.remote_field

    @property
    def name(self):
        return self.field.name

    @property
    def remote_name(self):
        return self.remote_field.name

    def __init__(self, subject, related, hint=None):
        self.subject = subject
        self.related = related
        self._hint = hint

    def __eq__(self, other):
        return type(self) is type(other) and self.related == other.related and self.subject == other.subject

    def __hash__(self):
        return hash((self.subject, self.related))

    def __repr__(self):
        return '<{}({}, {}, {})>'.format(self.__class__.__name__, self.name, self.subject, self.related)


class ChangeGraph:

    def __init__(self, data, disambiguate=True, namespace=None):
        self.nodes = []
        self.relations = {}
        self._lookup = {}
        self.namespace = namespace

        hints, relations = {}, set()
        for blob in copy.deepcopy(data):
            id, type = blob.pop('@id'), blob.pop('@type')

            self._lookup[id, type] = ChangeNode(self, id, type, blob, namespace=namespace)
            self.relations[self._lookup[id, type]] = set()
            self.nodes.append(self._lookup[id, type])

            for k, v in tuple(blob.items()):
                if isinstance(v, dict) and k != 'extra' and not k.startswith('@'):
                    related = (v.pop('@id'), v.pop('@type'))
                    hints[(id, type), related] = k
                    relations.add(((id, type), related))
                    blob.pop(k)
                if isinstance(v, list):
                    for rel in v:
                        subject = (rel.pop('@id'), rel.pop('@type'))
                        relations.add((subject, (id, type)))
                    blob.pop(k)

        for subject, related in relations:
            try:
                edge = GraphEdge(self._lookup[subject], self._lookup[related], hint=hints.get((subject, related)))
            except KeyError as e:
                raise UnresolvableReference(*e.args)

            self.relations[self._lookup[subject]].add(edge)
            self.relations[self._lookup[related]].add(edge)

        # if disambiguate:
        #     self.disambiguate()

        self.nodes = TopographicalSorter(self.nodes, dependencies=lambda n: tuple(e.related for e in n.related(backward=False))).sorted()

    def disambiguate(self):
        GraphDisambiguator().disambiguate(self)
        self.nodes = TopographicalSorter(self.nodes, dependencies=lambda n: tuple(e.related for e in n.related(backward=False))).sorted()

    def normalize(self):
        for node in self.nodes:
            # This feels overly hacky
            if hasattr(node.model, 'normalize'):
                node.model.normalize(node, self)

    def get(self, id, type):
        return self._lookup[(id, type)]

    def create(self, id, type, attrs):
        return self.add(ChangeNode(self, id or '_:{}'.format(uuid.uuid4()), type, attrs, namespace=self.namespace))

    def add(self, node):
        node.graph = self
        self.nodes.append(node)
        self.relations[node] = set()
        self._lookup[node.id, node.type] = node
        return node

    def relate(self, subject, related, hint=None):
        edge = GraphEdge(subject, related, hint)
        self.relations[subject].add(edge)
        self.relations[related].add(edge)
        return edge

    def replace(self, source, replacement):
        for edge in tuple(source.related()):
            # NOTE: Order of add & removes matters here due to
            # the hash function of an edge and how sets work
            self.relations[source].remove(edge)
            if edge.subject == source:
                self.relations[edge.related].remove(edge)
                edge.subject = replacement
                self.relations[edge.related].add(edge)
            else:
                self.relations[edge.subject].remove(edge)
                edge.related = replacement
                self.relations[edge.subject].add(edge)
{'fraction_non_alphanumeric': 0.11273209549071618, 'fraction_numerical': 0.000612119975515201, 'mean_word_length': 2.241732804232804, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2830411', 'n_tokens_mistral': 2782, 'n_tokens_neox': 2662, 'n_words': 686}