<commit_before>#!/usr/bin/env python2.7
from __future__ import print_function
import os
import sys
import tempfile
import subprocess

__author__ = '<NAME>'

# Use the bundled networkx egg
if __name__ == '__main__':
    nxegg = 'lib/networkx-1.7-py2.7.egg'
else:
    nxegg = os.path.join(os.path.dirname(__file__), 'lib/networkx-1.7-py2.7.egg')
sys.path.append(nxegg)

import csv
import networkx as nx
import xml.dom.minidom as minidom

from distutils.version import LooseVersion

if LooseVersion(nx.__version__) < LooseVersion('1.11'):
    write_dot = nx.write_dot
else:
    try:
        import pygraphviz
        from networkx.drawing.nx_agraph import write_dot
    except ImportError:
        try:
            if LooseVersion(nx.__version__) == LooseVersion('1.11'):
                import pydotplus
            else:
                import pydot
            from networkx.drawing.nx_pydot import write_dot
        except ImportError:
            from networkx.drawing.nx_agraph import write_dot

# Below constant is for changes introduced in NetworkX 2.1rc1, but
# LooseVersion thinks '2.1rc1' is higher than '2.1'. Change to '2.1rc1'
# when switching to pkg_resources.parse_version/packaging.version.parse.
_NX_BELOW_2_DOT_1 = (LooseVersion(nx.__version__) < LooseVersion('2.1'))

def pairs(lst, circular=False):
    """
    Loop through all pairs of successive items in a list.

    >>> list(pairs([1, 2, 3, 4]))
    [(1, 2), (2, 3), (3, 4)]
    >>> list(pairs([1, 2, 3, 4], circular=True))
    [(1, 2), (2, 3), (3, 4), (4, 1)]
    """
    i = iter(lst)
    first = prev = item = next(i)
    for item in i:
        yield prev, item
        prev = item
    if circular:
        yield item, first

def import_csv_graph(file):
    """
    Example:

    import_csv_graph(open("test.csv", "rb"))

    Does not handle multigraphs yet (more than 1 edge between the same 2 nodes).

    Each row must have the following (in this order):

    * Start node ID
    * End node ID
    * Length in meters
    * Edge name or ID
    * Start longitude, for example 18.4167
    * Start latitude, for example -33.9167
    * End longitude
    * End latitude
    """
    reader = csv.reader(file)
    graph = nx.Graph()
    for row_number, row in enumerate(reader):
        try:
            start_node = row[0]
            end_node = row[1]
            length = float(row[2])
            id = row[3]
            start_lon, start_lat, end_lon, end_lat = map(float, row[4:8])
            graph.add_edge(start_node, end_node, weight=length, id=id, label=id)

            # We keep the GPS coordinates as strings
            graph.node[start_node]['longitude'] = start_lon
            graph.node[start_node]['latitude'] = start_lat
            graph.node[end_node]['longitude'] = end_lon
            graph.node[end_node]['latitude'] = end_lat
        except ValueError:
            print("Skipping input row %d" % (row_number+1))

    return graph

def specify_positions(graph):
    lat_min = min([data['latitude'] for n, data in graph.nodes(data=True)])
    lat_max = max([data['latitude'] for n, data in graph.nodes(data=True)])
    lon_min = min([data['longitude'] for n, data in graph.nodes(data=True)])
    lon_max = max([data['longitude'] for n, data in graph.nodes(data=True)])

    for node, data in graph.nodes(data=True):
        latitude = data['latitude']
        longitude = data['longitude']
        y = (latitude - lat_min) / (lat_max - lat_min) * 1000
        x = (longitude - lon_min) / (lon_max - lon_min) * 1000
        graph.node[node]['pos'] = "%d,%d" % (int(x), int(y))



def make_png(graph, path):
    fd, dotfile = tempfile.mkstemp(prefix="graph", suffix=".dot")
    try:
        dfile = dotfile
        write_dot(graph, dfile)

        subprocess.call(['neato', '-n2', '-Tpng', '-o', path, dfile])
    finally:
        os.remove(dotfile)


def graph_components(graph):
    # The graph may contain multiple components, but we can only handle one connected component. If the graph contains
    # more than one connected component, we only use the largest one.
    components = list(nx.connected_component_subgraphs(graph))
    components.sort(key=lambda c: c.size(), reverse=True)

    return components



def odd_graph(graph):
    """
    Given a graph G, construct a graph containing only the vertices with odd degree from G. The resulting graph is
    fully connected, with each weight being the shortest path between the nodes in G.

    Complexity: O(V'*(E + V log(V)) )
    """
    result = nx.Graph()
    odd_nodes = [n for n in graph.nodes() if graph.degree(n) % 2 == 1]
    for u in odd_nodes:
        # We calculate the shortest paths twice here, but the overall performance hit is low
        paths = nx.shortest_path(graph, source=u, weight='weight')
        lengths = nx.shortest_path_length(graph, source=u, weight='weight')
        for v in odd_nodes:
            if u <= v:
                # We only add each edge once
                continue
            # The edge weights are negative for the purpose of max_weight_matching (we want the minimum weight)<commit_msg>Remove connected_component_subgraphs usage.<commit_after>#!/usr/bin/env python2.7
from __future__ import print_function
import os
import sys
import tempfile
import subprocess

__author__ = '<NAME>'

# Use the bundled networkx egg
if __name__ == '__main__':
    nxegg = 'lib/networkx-1.7-py2.7.egg'
else:
    nxegg = os.path.join(os.path.dirname(__file__), 'lib/networkx-1.7-py2.7.egg')
sys.path.append(nxegg)

import csv
import networkx as nx
import xml.dom.minidom as minidom

from distutils.version import LooseVersion

if LooseVersion(nx.__version__) < LooseVersion('1.11'):
    write_dot = nx.write_dot
else:
    try:
        import pygraphviz
        from networkx.drawing.nx_agraph import write_dot
    except ImportError:
        try:
            if LooseVersion(nx.__version__) == LooseVersion('1.11'):
                import pydotplus
            else:
                import pydot
            from networkx.drawing.nx_pydot import write_dot
        except ImportError:
            from networkx.drawing.nx_agraph import write_dot

# Below constant is for changes introduced in NetworkX 2.1rc1, but
# LooseVersion thinks '2.1rc1' is higher than '2.1'. Change to '2.1rc1'
# when switching to pkg_resources.parse_version/packaging.version.parse.
_NX_BELOW_2_DOT_1 = (LooseVersion(nx.__version__) < LooseVersion('2.1'))

def pairs(lst, circular=False):
    """
    Loop through all pairs of successive items in a list.

    >>> list(pairs([1, 2, 3, 4]))
    [(1, 2), (2, 3), (3, 4)]
    >>> list(pairs([1, 2, 3, 4], circular=True))
    [(1, 2), (2, 3), (3, 4), (4, 1)]
    """
    i = iter(lst)
    first = prev = item = next(i)
    for item in i:
        yield prev, item
        prev = item
    if circular:
        yield item, first

def import_csv_graph(file):
    """
    Example:

    import_csv_graph(open("test.csv", "rb"))

    Does not handle multigraphs yet (more than 1 edge between the same 2 nodes).

    Each row must have the following (in this order):

    * Start node ID
    * End node ID
    * Length in meters
    * Edge name or ID
    * Start longitude, for example 18.4167
    * Start latitude, for example -33.9167
    * End longitude
    * End latitude
    """
    reader = csv.reader(file)
    graph = nx.Graph()
    for row_number, row in enumerate(reader):
        try:
            start_node = row[0]
            end_node = row[1]
            length = float(row[2])
            id = row[3]
            start_lon, start_lat, end_lon, end_lat = map(float, row[4:8])
            graph.add_edge(start_node, end_node, weight=length, id=id, label=id)

            # We keep the GPS coordinates as strings
            graph.node[start_node]['longitude'] = start_lon
            graph.node[start_node]['latitude'] = start_lat
            graph.node[end_node]['longitude'] = end_lon
            graph.node[end_node]['latitude'] = end_lat
        except ValueError:
            print("Skipping input row %d" % (row_number+1))

    return graph

def specify_positions(graph):
    lat_min = min([data['latitude'] for n, data in graph.nodes(data=True)])
    lat_max = max([data['latitude'] for n, data in graph.nodes(data=True)])
    lon_min = min([data['longitude'] for n, data in graph.nodes(data=True)])
    lon_max = max([data['longitude'] for n, data in graph.nodes(data=True)])

    for node, data in graph.nodes(data=True):
        latitude = data['latitude']
        longitude = data['longitude']
        y = (latitude - lat_min) / (lat_max - lat_min) * 1000
        x = (longitude - lon_min) / (lon_max - lon_min) * 1000
        graph.node[node]['pos'] = "%d,%d" % (int(x), int(y))



def make_png(graph, path):
    fd, dotfile = tempfile.mkstemp(prefix="graph", suffix=".dot")
    try:
        dfile = dotfile
        write_dot(graph, dfile)

        subprocess.call(['neato', '-n2', '-Tpng', '-o', path, dfile])
    finally:
        os.remove(dotfile)


def graph_components(graph):
    # The graph may contain multiple components, but we can only handle one connected component. If the graph contains
    # more than one connected component, we only use the largest one.
    components = list(graph.subgraph(c) for c in nx.connected_components(graph))
    components.sort(key=lambda c: c.size(), reverse=True)

    return components



def odd_graph(graph):
    """
    Given a graph G, construct a graph containing only the vertices with odd degree from G. The resulting graph is
    fully connected, with each weight being the shortest path between the nodes in G.

    Complexity: O(V'*(E + V log(V)) )
    """
    result = nx.Graph()
    odd_nodes = [n for n in graph.nodes() if graph.degree(n) % 2 == 1]
    for u in odd_nodes:
        # We calculate the shortest paths twice here, but the overall performance hit is low
        paths = nx.shortest_path(graph, source=u, weight='weight')
        lengths = nx.shortest_path_length(graph, source=u, weight='weight')
        for v in odd_nodes:
            if u <= v:
                # We only add each edge once
                continue
            # The edge weights are negative for the purpose of max_weight_matching (we want the minimum weight)
{'fraction_non_alphanumeric': 0.11543729211504598, 'fraction_numerical': 0.016630796321659168, 'mean_word_length': 2.7242258652094717, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2422571', 'n_tokens_mistral': 3368, 'n_tokens_neox': 3206, 'n_words': 987}