<jupyter_start><jupyter_text>Convert Ensemble to Function Automatically.  Preliminaries  Imports<jupyter_code>import os
import pandas as pd
import sys
import networkx as nx

from os.path import dirname
from networkx.drawing.nx_pydot import to_pydot

note_dir = os.getcwd()

root_dir = dirname(note_dir)
src_dir = os.path.join(root_dir, "src")

sys.path.append(src_dir)

import morpheus
from morpheus.core import compose
from morpheus.tests import default_chain, default_ensemble, default_dataset
from morpheus.graph import (model_to_graph,
                            model_graph_traces,
                            model_graph_layout, 
                            convert_positions_to_dot_format,
                           add_merge_nodes)
from functools import partial
# In Greek to avoid nameclashes
ω = compose.o
χ = compose.x<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def fix_layout(G):
    
    for n in G.nodes():
        node = G.nodes(data=True)[n]
        if node['bipartite'] == 'func':
            node['shape'] = "\"square\""
            node['width'] = "2"
        elif node['bipartite'] == 'data':
            node['shape'] = "\"circle\""
        else:
            pass
    
    return G<jupyter_output><empty_output><jupyter_text>Ensemble<jupyter_code>data, _ = default_dataset(random_state=16)
e = default_ensemble(random_state=16)
data.values.shape
data[0]
e.pretty_print()<jupyter_output># Main
        Descriptive attributes:     [0. 1. 2. 3.]
        Target attributes:          [4. 5.]

        ## On types (mostly nominals)
        Target attribute types:                     ['numeric', 'numeric']
        N_classes of nominal target attributes:     []
        Classes of nominal target attributes:       []

        ## Weights
        Total weights of target attributes:         [1. 2.]

         
        ## Estimators
        Estimators used:        ['RandomForestRegressor', 'RandomForestRegressor']
        N_estimators:           2<jupyter_text>Conversion<jupyter_code>model_one = e.estimators_[0]
model_two = e.estimators_[1]
G_one = model_to_graph(model_one, idx=1)
G_two = model_to_graph(model_two, idx=2)
G = nx.compose(G_one, G_two)
G = fix_layout(G)
X = to_pydot(G)
X.set('rankdir', 'BT')
X.set('splines', 'ortho')

with open("tmp/test.dot", "w") as text_file:
    print(X.to_string(), file=text_file)
    
!dot -T png tmp/test.dot > tmp/test.png  # Bash command

from IPython.display import Image, display
from IPython.core.display import HTML 

display(Image('tmp/test.png', unconfined=True))
add_merge_nodes(G)
X = to_pydot(G)
X.set('rankdir', 'BT')
X.set('splines', 'ortho')

with open("tmp/test.dot", "w") as text_file:
    print(X.to_string(), file=text_file)
    
!dot -T png tmp/test.dot > tmp/test.png  # Bash command

from IPython.display import Image, display
from IPython.core.display import HTML 

display(Image('tmp/test.png', unconfined=True))
import numpy as np

def select(idx):
    
    def select(X):
        if len(X.shape) > 1:
            return X[:, idx]
        elif len(X.shape) == 1:
            assert idx == 0
            return X
    return select
# Convert the graph to its functions
sorted_list = list(nx.topological_sort(G))
print(sorted_list)
functions = {}
for node_name in sorted_list:
    node = G.nodes(data=True)[node_name]
    #print(node)
    if node.get('kind', None) == 'data': 
        if len(nx.ancestors(G, node_name)) == 0:
            functions[node_name] = select(node['idx'])
        elif len(nx.descendants(G, node_name)) == 0:
            functions[node_name] = 'end'
            
    elif node.get('kind', None) == 'model':
        
        previous_nodes = [t[0] for t in G.in_edges(node_name)]
        
        inputs = [functions[n] for n in previous_nodes]
        inputs = ω(np.transpose, χ(*inputs, return_type=np.array))
        f = node['function']
        functions[node_name] = ω(f, inputs)
    
    elif node.get('kind', None) == 'merge':
        merge_idx = node['idx']
        previous_nodes = [t[0] for t in G.in_edges(node_name)]
        previous_t_idx = [G.nodes()[n]['tgt'] for n in previous_nodes]
        print(previous_t_idx)
        inputs = [(functions[n],t) for n, t in zip(previous_nodes, previous_t_idx)]
        
        inputs = [ω(select(t_idx.index(merge_idx)), f) for f, t_idx in inputs]
        inputs = ω(np.transpose, χ(*inputs, return_type=np.array))
        
        f = partial(np.mean, axis=1)
        functions[node_name] = ω(f, inputs)
functions
res = functions['M(d-05)'](data.values)

res.shape
res
functions['f-01'](data.values)
functions['f-02'](data.values)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12704565030146425, 'fraction_numerical': 0.0071059431524547806, 'mean_word_length': 2.8388429752066116, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '530969', 'license': 'MIT', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1571, 'n_words': 352, 'path': 'note/dev-1905/190424 - convert ensemble to function.ipynb', 'repo_name': 'eliavw/morpheus', 'size': '4644'}