<commit_before>import datetime
from itertools import chain, imap
from sets import Set

from django.contrib.contenttypes.models import ContentType
from django.utils.html import linebreaks

from valuenetwork.valueaccounting.models import Commitment, Process

def split_thousands(n, sep=','):
    s = str(n)
    if len(s) <= 3: return s  
    return split_thousands(s[:-3], sep) + sep + s[-3:]

def dfs(node, all_nodes, depth):
    """
    Performs a recursive depth-first search starting at ``node``. 
    """
    to_return = [node,]
    for subnode in all_nodes:
        if subnode.parent and subnode.parent.id == node.id:
            to_return.extend(dfs(subnode, all_nodes, depth+1))
    return to_return

def flattened_children(node, all_nodes, to_return):
     to_return.append(node)
     for subnode in all_nodes:
         if subnode.parent and subnode.parent.id == node.id:
             flattened_children(subnode, all_nodes, to_return)
     return to_return

class Edge(object):
    def __init__(self, from_node, to_node, label):
        self.from_node = from_node
        self.to_node = to_node
        self.label = label
        self.width = 1

    def dictify(self):
        d = {
            "from_node": self.from_node.node_id(),
            "to_node": self.to_node.node_id(),
            "label": self.label,
            "width": self.width,
        }
        return d
            

def process_graph(processes):
    nodes = []
    visited = Set()
    edges = []
    for p in processes:
        if p not in visited:
            visited.add(p)
            d = {
                "id": p.node_id(),
                "name": p.name,
                "start": p.start_date.strftime('%Y-%m-%d'),
                "end": p.end_date.strftime('%Y-%m-%d'),
                }
            nodes.append(d)
        next = p.next_processes()
        for n in next:
            if n not in visited:
                visited.add(n)
                d = {
                    "id": n.node_id(),
                    "name": n.name,
                    "start": n.start_date.strftime('%Y-%m-%d'),
                    "end": n.end_date.strftime('%Y-%m-%d'),
                    }
                nodes.append(d)
            edge = Edge(p, n, "project")
            edges.append(edge.dictify())
    big_d = {
        "nodes": nodes,
        "edges": edges,
    }
    return big_d
            
def project_graph(producers):
    nodes = []
    edges = []
    #import pdb; pdb.set_trace()
    for p in producers:
        for rt in p.produced_resource_type_relationships():
            for pt in rt.resource_type.consuming_process_type_relationships():
                if p.project and pt.process_type.project:
                    if p.project != pt.process_type.project:
                        nodes.extend([p.project, pt.process_type.project, rt.resource_type])
                        edges.append(Edge(p.project, rt.resource_type, rt.event_type.label))
                        edges.append(Edge(rt.resource_type, pt.process_type.project, pt.inverse_label()))
    return [nodes, edges]

def explode(process_type_relationship, nodes, edges, depth, depth_limit):
    if depth > depth_limit:
        return
    #if process_type_relationship.process_type.name.startswith('Q'):
    #    return<commit_msg>add previous processes to json-processes graph<commit_after>import datetime
from itertools import chain, imap
from sets import Set

from django.contrib.contenttypes.models import ContentType
from django.utils.html import linebreaks

from valuenetwork.valueaccounting.models import Commitment, Process

def split_thousands(n, sep=','):
    s = str(n)
    if len(s) <= 3: return s  
    return split_thousands(s[:-3], sep) + sep + s[-3:]

def dfs(node, all_nodes, depth):
    """
    Performs a recursive depth-first search starting at ``node``. 
    """
    to_return = [node,]
    for subnode in all_nodes:
        if subnode.parent and subnode.parent.id == node.id:
            to_return.extend(dfs(subnode, all_nodes, depth+1))
    return to_return

def flattened_children(node, all_nodes, to_return):
     to_return.append(node)
     for subnode in all_nodes:
         if subnode.parent and subnode.parent.id == node.id:
             flattened_children(subnode, all_nodes, to_return)
     return to_return

class Edge(object):
    def __init__(self, from_node, to_node, label):
        self.from_node = from_node
        self.to_node = to_node
        self.label = label
        self.width = 1

    def dictify(self):
        d = {
            "from_node": self.from_node.node_id(),
            "to_node": self.to_node.node_id(),
            "label": self.label,
            "width": self.width,
        }
        return d
            

def process_graph(processes):
    nodes = []
    visited = Set()
    edges = []
    for p in processes:
        if p not in visited:
            visited.add(p)
            d = {
                "id": p.node_id(),
                "name": p.name,
                "start": p.start_date.strftime('%Y-%m-%d'),
                "end": p.end_date.strftime('%Y-%m-%d'),
                }
            nodes.append(d)
        next = p.next_processes()
        for n in next:
            if n not in visited:
                visited.add(n)
                d = {
                    "id": n.node_id(),
                    "name": n.name,
                    "start": n.start_date.strftime('%Y-%m-%d'),
                    "end": n.end_date.strftime('%Y-%m-%d'),
                    }
                nodes.append(d)
            edge = Edge(p, n, "project-link")
            edges.append(edge.dictify())
        prev = p.previous_processes()
        for n in prev:
            if n not in visited:
                visited.add(n)
                d = {
                    "id": n.node_id(),
                    "name": n.name,
                    "start": n.start_date.strftime('%Y-%m-%d'),
                    "end": n.end_date.strftime('%Y-%m-%d'),
                    }
                nodes.append(d)
            edge = Edge(n, n, "project-link")
            edges.append(edge.dictify())
    big_d = {
        "nodes": nodes,
        "edges": edges,
    }
    return big_d
            
def project_graph(producers):
    nodes = []
    edges = []
    #import pdb; pdb.set_trace()
    for p in producers:
        for rt in p.produced_resource_type_relationships():
            for pt in rt.resource_type.consuming_process_type_relationships():
                if p.project and pt.process_type.project:
                    if p.project != pt.process_type.project:
                        nodes.extend([p.project, pt.process_type.project, rt.resource_type])
                        edges.append(Edge(p.project, rt.resource_type, rt.event_type.label))
                        edges.append(Edge(rt.resource_type, pt.process_type.project, pt.inverse_label()))
    return [nodes, edges]

def explode(process_type_relationship, nodes, edges, depth, depth_limit):
    if depth > depth_limit:
        return
    #if process_type_relationship.process_type.name.startswith('Q'):
    #    return
{'fraction_non_alphanumeric': 0.11853417899929528, 'fraction_numerical': 0.0014094432699083862, 'mean_word_length': 1.8181096108022239, 'pattern_counts': {'":': 32, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2476927', 'n_tokens_mistral': 2219, 'n_tokens_neox': 2137, 'n_words': 481}