<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""Defines functionality for pipelined execution of interfaces

The `Pipeline` class provides core functionality for batch processing. 
"""

from copy import deepcopy
import logging
import os
import re

from nipype.utils.misc import package_check
package_check('networkx', '1.0')
import networkx as nx

from nipype.interfaces.base import CommandLine, isdefined
from nipype.utils.filemanip import fname_presuffix, FileNotFoundError

logger = logging.getLogger('workflow')

try:
   from os.path import relpath
except ImportError:
   import os
   import os.path as op
   def relpath(path, start=None):
       """Return a relative version of a path"""
       if start is None:
           start = os.curdir
       if not path:
           raise ValueError("no path specified")
       start_list = op.abspath(start).split(op.sep)
       path_list = op.abspath(path).split(op.sep)
       if start_list[0].lower() != path_list[0].lower():
           unc_path, rest = op.splitunc(path)
           unc_start, rest = op.splitunc(start)
           if bool(unc_path) ^ bool(unc_start):
               raise ValueError("Cannot mix UNC and non-UNC paths (%s and%s)" %
                                                                   (path, start))
           else:
               raise ValueError("path is on drive %s, start on drive %s"
                                            % (path_list[0], start_list[0]))
       # Work out how much of the filepath is shared by start and path.
       for i in range(min(len(start_list), len(path_list))):
           if start_list[i].lower() != path_list[i].lower():
               break
       else:
           i += 1

       rel_list = [op.pardir] * (len(start_list)-i) + path_list[i:]
       if not rel_list:
           return os.curdir
       return op.join(*rel_list)

def walk(children, level=0, path=None, usename=True):
    """Generate all the full paths in a tree, as a dict.
    """
    # Entry point
    if level == 0:
        path = {}
    # Exit condition
    if not children:
        yield path.copy()
        return
    # Tree recursion
    head, tail = children[0], children[1:]
    name, func = head
    for child in func():
        # We can use the arg name or the tree level as a key
        if usename:
            path[name] = child
        else:
            path[level] = child
        # Recurse into the next level
        for child_paths in walk(tail, level+1, path, usename):
            yield child_paths
        
def _create_pickleable_graph(graph, show_connectinfo=False):
    """Create a graph that can be pickled.

    Ensures that edge info is pickleable.
    """
    logger.debug('creating pickleable graph')
    pklgraph = deepcopy(graph)
    for edge in pklgraph.edges():
        data = pklgraph.get_edge_data(*edge)
        pklgraph.remove_edge(*edge)
        if show_connectinfo:
            pklgraph.add_edge(edge[0], edge[1], l=str(data['connect']))
        else:
            pklgraph.add_edge(edge[0], edge[1])
    return pklgraph

def _create_dot_graph(graph, show_connectinfo=False):
    """Create a graph that can be pickled.

    Ensures that edge info is pickleable.
    """
    logger.debug('creating pickleable graph')
    pklgraph = nx.DiGraph()
    for edge in graph.edges():
        data = graph.get_edge_data(*edge)
        if hasattr(edge[0], '_interface'):
            srcclass = edge[0]._interface.__class__.__module__.split('.')[2]
        else:
            srcclass = ''
        srcname = '.'.join(str(edge[0]).split('.')[1:])
        srcname = '.'.join((srcname, srcclass))
        if hasattr(edge[1], '_interface'):
            destclass = edge[1]._interface.__class__.__module__.split('.')[2]
        else:
            destclass = ''
        destname = '.'.join(str(edge[1]).split('.')[1:])
        destname = '.'.join((destname, destclass))
        if show_connectinfo:
            pklgraph.add_edge(srcname, destname, l=str(data['connect']))
        else:
            pklgraph.add_edge(srcname, destname)
    return pklgraph

def _write_detailed_dot(graph, dotfilename):
    """Create a dot file with connection info

    digraph structs {
    node [shape=record];
    struct1 [label="<f0> left|<f1> mid\ dle|<f2> right"];
    struct2 [label="<f0> one|<f1> two"];
    struct3 [label="hello\nworld |{ b |{c|<here> d|e}| f}| g | h"];
    struct1:f1 -> struct2:f0;
    struct1:f0 -> struct2:f1;
    struct1:f2 -> struct3:here;
    }
    """
    text = ['digraph structs {', 'node [shape=record];']
    # write nodes
    edges = []
    replacefunk = lambda x: x.replace('_', '').replace('.', ''). \
        replace('@', '').replace('-', '')
    for n in graph.nodes():
        nodename = str(n)
        inports = []
        for u, v, d in graph.in_edges_iter(nbunch=n, data=True):
            for cd in d['connect']:
                if isinstance(cd[0], str):
                    outport = cd[0]
                else:
                    outport = cd[0][0]
                inport = cd[1]
                ipstrip = 'in'+replacefunk(inport)
                opstrip = 'out'+replacefunk(outport)
                edges.append('%s:%s:e -> %s:%s:w;' % (str(u).replace('.', ''),
                                                  opstrip,
                                                  str(v).replace('.', ''),
                                                  ipstrip))
                if inport not in inports:
                    inports.append(inport)
        inputstr = '{IN'
        for ip in inports:
            inputstr += '|<in%s> %s' % (replacefunk(ip), ip)
        inputstr += '}'
        outports = []
        for u, v, d in graph.out_edges_iter(nbunch=n, data=True):
            for cd in d['connect']:
                if isinstance(cd[0], str):
                    outport = cd[0]
                else:
                    outport = cd[0][0]
                if outport not in outports:
                    outports.append(outport)
        outputstr = '{OUT'
        for op in outports:
            outputstr += '|<out%s> %s' % (replacefunk(op), op)
        outputstr += '}'
        if hasattr(n, '_interface'):
            srcpackage = n._interface.__class__.__module__.split('.')[2]
        else:
            srcpackage = ''
        srchierarchy = '.'.join(nodename.split('.')[1:-1])
        nodenamestr = '{ %s | %s | %s }'% (nodename.split('.')[-1], srcpackage, srchierarchy)
        text += ['%s [label="%s|%s|%s"];' % (nodename.replace('.', ''),
                                             inputstr, nodenamestr,
                                             outputstr)]
    # write edges
    for edge in edges:
        text.append(edge)
    text.append('}')
    filep = open(dotfilename, 'wt')
    filep.write('\n'.join(text))
    filep.close()
    return text
<commit_msg>fixed graph generation when using nodes outside of nipype code tree<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""Defines functionality for pipelined execution of interfaces

The `Pipeline` class provides core functionality for batch processing. 
"""

from copy import deepcopy
import logging
import os
import re

from nipype.utils.misc import package_check
package_check('networkx', '1.0')
import networkx as nx

from nipype.interfaces.base import CommandLine, isdefined
from nipype.utils.filemanip import fname_presuffix, FileNotFoundError

logger = logging.getLogger('workflow')

try:
   from os.path import relpath
except ImportError:
   import os
   import os.path as op
   def relpath(path, start=None):
       """Return a relative version of a path"""
       if start is None:
           start = os.curdir
       if not path:
           raise ValueError("no path specified")
       start_list = op.abspath(start).split(op.sep)
       path_list = op.abspath(path).split(op.sep)
       if start_list[0].lower() != path_list[0].lower():
           unc_path, rest = op.splitunc(path)
           unc_start, rest = op.splitunc(start)
           if bool(unc_path) ^ bool(unc_start):
               raise ValueError("Cannot mix UNC and non-UNC paths (%s and%s)" %
                                                                   (path, start))
           else:
               raise ValueError("path is on drive %s, start on drive %s"
                                            % (path_list[0], start_list[0]))
       # Work out how much of the filepath is shared by start and path.
       for i in range(min(len(start_list), len(path_list))):
           if start_list[i].lower() != path_list[i].lower():
               break
       else:
           i += 1

       rel_list = [op.pardir] * (len(start_list)-i) + path_list[i:]
       if not rel_list:
           return os.curdir
       return op.join(*rel_list)

def walk(children, level=0, path=None, usename=True):
    """Generate all the full paths in a tree, as a dict.
    """
    # Entry point
    if level == 0:
        path = {}
    # Exit condition
    if not children:
        yield path.copy()
        return
    # Tree recursion
    head, tail = children[0], children[1:]
    name, func = head
    for child in func():
        # We can use the arg name or the tree level as a key
        if usename:
            path[name] = child
        else:
            path[level] = child
        # Recurse into the next level
        for child_paths in walk(tail, level+1, path, usename):
            yield child_paths
        
def _create_pickleable_graph(graph, show_connectinfo=False):
    """Create a graph that can be pickled.

    Ensures that edge info is pickleable.
    """
    logger.debug('creating pickleable graph')
    pklgraph = deepcopy(graph)
    for edge in pklgraph.edges():
        data = pklgraph.get_edge_data(*edge)
        pklgraph.remove_edge(*edge)
        if show_connectinfo:
            pklgraph.add_edge(edge[0], edge[1], l=str(data['connect']))
        else:
            pklgraph.add_edge(edge[0], edge[1])
    return pklgraph

def _create_dot_graph(graph, show_connectinfo=False):
    """Create a graph that can be pickled.

    Ensures that edge info is pickleable.
    """
    logger.debug('creating pickleable graph')
    pklgraph = nx.DiGraph()
    for edge in graph.edges():
        data = graph.get_edge_data(*edge)
        if hasattr(edge[0], '_interface'):
            pkglist = edge[0]._interface.__class__.__module__.split('.')
            if len(pkglist) > 1:
                srcclass = pkglist[-2]
            else:
                srcclass = ''
        else:
            srcclass = ''
        srcname = '.'.join(str(edge[0]).split('.')[1:])
        srcname = '.'.join((srcname, srcclass))
        if hasattr(edge[1], '_interface'):
            pkglist = edge[1]._interface.__class__.__module__.split('.')
            if len(pkglist) > 1:
                destclass = pkglist[-2]
            else:
                destclass = ''
        else:
            destclass = ''
        destname = '.'.join(str(edge[1]).split('.')[1:])
        destname = '.'.join((destname, destclass))
        if show_connectinfo:
            pklgraph.add_edge(srcname, destname, l=str(data['connect']))
        else:
            pklgraph.add_edge(srcname, destname)
    return pklgraph

def _write_detailed_dot(graph, dotfilename):
    """Create a dot file with connection info

    digraph structs {
    node [shape=record];
    struct1 [label="<f0> left|<f1> mid\ dle|<f2> right"];
    struct2 [label="<f0> one|<f1> two"];
    struct3 [label="hello\nworld |{ b |{c|<here> d|e}| f}| g | h"];
    struct1:f1 -> struct2:f0;
    struct1:f0 -> struct2:f1;
    struct1:f2 -> struct3:here;
    }
    """
    text = ['digraph structs {', 'node [shape=record];']
    # write nodes
    edges = []
    replacefunk = lambda x: x.replace('_', '').replace('.', ''). \
        replace('@', '').replace('-', '')
    for n in graph.nodes():
        nodename = str(n)
        inports = []
        for u, v, d in graph.in_edges_iter(nbunch=n, data=True):
            for cd in d['connect']:
                if isinstance(cd[0], str):
                    outport = cd[0]
                else:
                    outport = cd[0][0]
                inport = cd[1]
                ipstrip = 'in'+replacefunk(inport)
                opstrip = 'out'+replacefunk(outport)
                edges.append('%s:%s:e -> %s:%s:w;' % (str(u).replace('.', ''),
                                                  opstrip,
                                                  str(v).replace('.', ''),
                                                  ipstrip))
                if inport not in inports:
                    inports.append(inport)
        inputstr = '{IN'
        for ip in inports:
            inputstr += '|<in%s> %s' % (replacefunk(ip), ip)
        inputstr += '}'
        outports = []
        for u, v, d in graph.out_edges_iter(nbunch=n, data=True):
            for cd in d['connect']:
                if isinstance(cd[0], str):
                    outport = cd[0]
                else:
                    outport = cd[0][0]
                if outport not in outports:
                    outports.append(outport)
        outputstr = '{OUT'
        for op in outports:
            outputstr += '|<out%s> %s' % (replacefunk(op), op)
        outputstr += '}'
        if hasattr(n, '_interface'):
            pkglist = n._interface.__class__.__module__.split('.')
            if len(pkglist) > 1:
                srcpackage = pkglist[-2]
            else:
                srcpackage = ""
        else:
            srcpackage = ''
        srchierarchy = '.'.join(nodename.split('.')[1:-1])
        nodenamestr = '{ %s | %s | %s }'% (nodename.split('.')[-1], srcpackage, srchierarchy)
        text += ['%s [label="%s|%s|%s"];' % (nodename.replace('.', ''),
                                             inputstr, nodenamestr,
                                             outputstr)]
    # write edges
    for edge in edges:
        text.append(edge)
    text.append('}')
    filep = open(dotfilename, 'wt')
    filep.write('\n'.join(text))
    filep.close()
    return text

{'fraction_non_alphanumeric': 0.13289223622158092, 'fraction_numerical': 0.008939255296684732, 'mean_word_length': 1.8581774290887145, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2453416', 'n_tokens_mistral': 4350, 'n_tokens_neox': 4283, 'n_words': 1103}