<commit_before># -*- coding: utf-8 -*-

import copy
import logging as logger

from graph_storage_drivers.graph_dict import DictStorage


def adjacency_to_edges(nodes, adjacency, node_source):
    """
    Construct edges for nodes based on adjacency.

    Edges are created for every node in `nodes` based on the neighbors of
    the node in adjacency if the neighbor node is also in `node_source`.
    The source of adjacency information would normally be self.graph and
    self.nodes for `node_source`. However, `node_source may` also equal
    `nodes` to return edges for the isolated graph.

    :param nodes:       nodes to return edges for
    :type nodes:        list
    :param adjacency:   node adjacency (self.graph)
    :type adjacency:    dict
    :param node_source: other nodes to consider when creating edges
    :type node_source:  list
    """

    edges = []
    for node in nodes:
        edges.extend([tuple([node, e]) for e in adjacency[node] if e in node_source])

    return edges


def edge_list_to_adjacency(edges):
    """
    Create adjacency dictionary based on a list of edges

    :param edges: edges to create adjacency for
    :type edges:  list
    """

    adjacency = dict([(n, []) for n in edge_list_to_nodes(edges)])
    for edge in edges:
        adjacency[edge[0]].append(edge[1])

    return adjacency


def edge_list_to_nodes(edges):
    """
    Create a list of nodes from a list of edges

    :param edges: edges to create nodes for
    :type edges:  list
    """

    return list(set(sum(edges, ())))


def make_edges(nodes, directed=True):
    """
    Create an edge tuple from two nodes either directed
    (first to second) or undirected (two edges, both ways).

    :param nodes:    nodes to create edges for
    :type nodes:     list or tuple
    :param directed: greate directed edge or not
    :type directed:  bool
    """

    edges = [tuple(nodes)]
    if not directed:
        edges.append(nodes[::-1])

    return edges


def renumber_id(graph, start):
    """
    Renumber all node ID's in the graph from a new start ID and adjust edges
    accordingly. Useful when duplicating a graph substructure.
    If the graph uses auto_nid, the node nid is also changed.

    #TODO: this one failes if run on a subgraph. Probably need to make changes
    #to nids in place instead of registering new DictStorage

    :param graph:   Graph object to renumber
    :type graph:    Graph object
    :param start:   New start number to renumber from
    :type start:    :py:int
    :return:        Renumber graph and mapping of old to new ID
    :rtype:         Graph object, :py:dict
    """

    start = copy.copy(start)
    mapper = {}
    for nid, value in sorted(graph.nodes.items()):
        mapper[value['_id']] = start

        # Renumber
        value['_id'] = start
        if graph.auto_nid:
            value['nid'] = start

        start += 1

    # Update nid if auto_nid
    if graph.auto_nid:
        newnodes = {v: graph.nodes[k] for k, v in mapper.items()}
        graph.nodes = DictStorage(newnodes)

    # Update root
    if graph.root:
        graph.root = mapper[graph.root]

    # Update edges.
    newedges = {}
    for eid, edge in graph.edges.items():
        newedges[(mapper.get(eid[0], eid[0]), mapper.get(eid[1], eid[1]))] = edge<commit_msg>Remove nid from node attributes, no longer in use<commit_after># -*- coding: utf-8 -*-

import copy
import logging as logger

from graph_storage_drivers.graph_dict import DictStorage


def adjacency_to_edges(nodes, adjacency, node_source):
    """
    Construct edges for nodes based on adjacency.

    Edges are created for every node in `nodes` based on the neighbors of
    the node in adjacency if the neighbor node is also in `node_source`.
    The source of adjacency information would normally be self.graph and
    self.nodes for `node_source`. However, `node_source may` also equal
    `nodes` to return edges for the isolated graph.

    :param nodes:       nodes to return edges for
    :type nodes:        list
    :param adjacency:   node adjacency (self.graph)
    :type adjacency:    dict
    :param node_source: other nodes to consider when creating edges
    :type node_source:  list
    """

    edges = []
    for node in nodes:
        edges.extend([tuple([node, e]) for e in adjacency[node] if e in node_source])

    return edges


def edge_list_to_adjacency(edges):
    """
    Create adjacency dictionary based on a list of edges

    :param edges: edges to create adjacency for
    :type edges:  list
    """

    adjacency = dict([(n, []) for n in edge_list_to_nodes(edges)])
    for edge in edges:
        adjacency[edge[0]].append(edge[1])

    return adjacency


def edge_list_to_nodes(edges):
    """
    Create a list of nodes from a list of edges

    :param edges: edges to create nodes for
    :type edges:  list
    """

    return list(set(sum(edges, ())))


def make_edges(nodes, directed=True):
    """
    Create an edge tuple from two nodes either directed
    (first to second) or undirected (two edges, both ways).

    :param nodes:    nodes to create edges for
    :type nodes:     list or tuple
    :param directed: greate directed edge or not
    :type directed:  bool
    """

    edges = [tuple(nodes)]
    if not directed:
        edges.append(nodes[::-1])

    return edges


def renumber_id(graph, start):
    """
    Renumber all node ID's in the graph from a new start ID and adjust edges
    accordingly. Useful when duplicating a graph substructure.
    If the graph uses auto_nid, the node nid is also changed.

    #TODO: this one failes if run on a subgraph. Probably need to make changes
    #to nids in place instead of registering new DictStorage

    :param graph:   Graph object to renumber
    :type graph:    Graph object
    :param start:   New start number to renumber from
    :type start:    :py:int
    :return:        Renumber graph and mapping of old to new ID
    :rtype:         Graph object, :py:dict
    """

    start = copy.copy(start)
    mapper = {}
    for nid, value in sorted(graph.nodes.items()):
        mapper[value['_id']] = start

        # Renumber
        value['_id'] = start
        start += 1

    # Update nid if auto_nid
    if graph.auto_nid:
        newnodes = {v: graph.nodes[k] for k, v in mapper.items()}
        graph.nodes = DictStorage(newnodes)

    # Update root
    if graph.root:
        graph.root = mapper[graph.root]

    # Update edges.
    newedges = {}
    for eid, edge in graph.edges.items():
        newedges[(mapper.get(eid[0], eid[0]), mapper.get(eid[1], eid[1]))] = edge
{'fraction_non_alphanumeric': 0.08995910949568378, 'fraction_numerical': 0.0027260336210813267, 'mean_word_length': 2.685267857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '246057', 'n_tokens_mistral': 2045, 'n_tokens_neox': 1959, 'n_words': 761}