<commit_before>from __future__ import absolute_import, division, print_function

import networkx as nx
from dask.core import istask, get_dependencies


def make_hashable(x):
    try:
        hash(x)
        return x
    except TypeError:
        return hash(str(x))


def lower(func):
    while hasattr(func, 'func'):
        func = func.func
    return func

def name(func):
    try:
        return lower(func).__name__
    except AttributeError:
        return 'func'


def to_networkx(d, data_attributes=None, function_attributes=None):
    if data_attributes is None:
        data_attributes = dict()
    if function_attributes is None:
        function_attributes = dict()

    g = nx.DiGraph()

    for k, v in sorted(d.items(), key=lambda x: x[0]):
        g.add_node(k, shape='box', **data_attributes.get(k, dict()))
        if istask(v):
            func, args = v[0], v[1:]
            func_node = make_hashable((v, 'function'))
            g.add_node(func_node,
                       shape='circle',
                       label=name(func),
                       **function_attributes.get(k, dict()))
            g.add_edge(func_node, k)
            for dep in sorted(get_dependencies(d, k)):
                arg2 = make_hashable(dep)
                g.add_node(arg2,
                           label=str(dep),
                           shape='box',
                           **data_attributes.get(dep, dict()))
                g.add_edge(arg2, func_node)
        else:
            if v not in d:
                g.add_node(k, label='%s=%s' % (k, v), **data_attributes.get(k, dict()))
            else:  # alias situation
                g.add_edge(v, k)

    return g


def write_networkx_to_dot(dg, filename='mydask'):
    import os
    p = nx.to_pydot(dg)
    p.set_rankdir('BT')
    with open(filename + '.dot', 'w') as f:<commit_msg>raise informative error when pydot is missing<commit_after>from __future__ import absolute_import, division, print_function

import networkx as nx
from dask.core import istask, get_dependencies


def make_hashable(x):
    try:
        hash(x)
        return x
    except TypeError:
        return hash(str(x))


def lower(func):
    while hasattr(func, 'func'):
        func = func.func
    return func

def name(func):
    try:
        return lower(func).__name__
    except AttributeError:
        return 'func'


def to_networkx(d, data_attributes=None, function_attributes=None):
    if data_attributes is None:
        data_attributes = dict()
    if function_attributes is None:
        function_attributes = dict()

    g = nx.DiGraph()

    for k, v in sorted(d.items(), key=lambda x: x[0]):
        g.add_node(k, shape='box', **data_attributes.get(k, dict()))
        if istask(v):
            func, args = v[0], v[1:]
            func_node = make_hashable((v, 'function'))
            g.add_node(func_node,
                       shape='circle',
                       label=name(func),
                       **function_attributes.get(k, dict()))
            g.add_edge(func_node, k)
            for dep in sorted(get_dependencies(d, k)):
                arg2 = make_hashable(dep)
                g.add_node(arg2,
                           label=str(dep),
                           shape='box',
                           **data_attributes.get(dep, dict()))
                g.add_edge(arg2, func_node)
        else:
            if v not in d:
                g.add_node(k, label='%s=%s' % (k, v), **data_attributes.get(k, dict()))
            else:  # alias situation
                g.add_edge(v, k)

    return g


def write_networkx_to_dot(dg, filename='mydask'):
    import os
    try:
        p = nx.to_pydot(dg)
    except AttributeError:
        raise ImportError("Can not find pydot module. Please install.\n"
                          "    pip install pydot")
    p.set_rankdir('BT')
    with open(filename + '.dot', 'w') as f:
{'fraction_non_alphanumeric': 0.1150691952844695, 'fraction_numerical': 0.0030753459764223477, 'mean_word_length': 1.8783185840707965, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5764428', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1177, 'n_words': 291}