<commit_before># Licensed Materials - Property of IBM
# Copyright IBM Corp. 2015

import uuid
import json
import inspect
import pickle
import base64
import streamsx.topology.dependency
from streamsx.topology.schema import CommonSchema

class SPLGraph(object):

    def __init__(self, name=None):
        if name is None:
            name = str(uuid.uuid1()).replace("-", "")
        self.name = name
        self.operators = []
        self.resolver = streamsx.topology.dependency.DependencyResolver()

    def addOperator(self, kind, function=None, name=None):
        if name is None:
            if function is not None:
               if hasattr(function, '__name__'):
                   name = function.__name__ + "_" + str(len(self.operators))
               elif hasattr(function, '__class__'):
                   name = function.__class__.__name__ + "_" + str(len(self.operators))
            else:
               name = self.name + "_OP"+str(len(self.operators))
        if(kind.startswith("$")):    
            op = Marker(len(self.operators), kind, name, {}, self)                           
        else:
            op = SPLInvocation(len(self.operators), kind, function, name, {}, self)
        self.operators.append(op)
        if not function is None:
            if not inspect.isbuiltin(function):
                self.resolver.add_dependencies(inspect.getmodule(function))
        return op
    
    def addPassThruOperator(self):
        name = self.name + "_OP"+str(len(self.operators))
        op = SPLInvocation(len(self.operators), "com.ibm.streamsx.topology.functional.python::PyFunctionPassThru", None, name, {}, self)
        self.operators.append(op)
        return op

    def generateSPLGraph(self):
        _graph = {}
        _graph["name"] = self.name
        _graph["namespace"] = self.name
        _graph["public"] = True
        _graph["config"] = {}
        _graph["config"]["includes"] = []
        _ops = []
        self.addModules(_graph["config"]["includes"])
        self.addPackages(_graph["config"]["includes"])
        for op in self.operators:
            _ops.append(op.generateSPLOperator())

        _graph["operators"] = _ops
        return _graph
   
    def addPackages(self, includes):
        for package_path in self.resolver.packages:
           mf = {}
           mf["source"] = package_path
           mf["target"] = "opt/python/packages"
           includes.append(mf)

    def addModules(self, includes):
        for module_path in self.resolver.modules:
           mf = {}
           mf["source"] = module_path
           mf["target"] = "opt/python/modules"
           includes.append(mf)
           
    def getLastOperator(self):
        return self.operators[len(self.operators) -1]      
        
    def printJSON(self):
      print(json.dumps(self.generateSPLGraph(), sort_keys=True, indent=4, separators=(',', ': ')))

class SPLInvocation(object):

    def __init__(self, index, kind, function, name, params, graph):
        self.index = index
        self.kind = kind
        self.function = function
        self.name = name
        self.params = {}
        self.setParameters(params)
        self._addOperatorFunction(self.function)
        self.graph = graph

        self.inputPorts = []
        self.outputPorts = []

    def addOutputPort(self, oWidth=None, name=None, inputPort=None, schema= CommonSchema.Python):
        if name is None:
            name = self.name + "_OUT"+str(len(self.outputPorts))
        oport = OPort(name, self, len(self.outputPorts), schema, oWidth)
        self.outputPorts.append(oport)

        if not inputPort is None:
            oport.connect(inputPort)
        return oport

    def setParameters(self, params):
        for param in params:
            self.params[param] = params[param]

    def appendParameters(self, params):
        for param in params:
            if self.params.get(param) is None:
                self.params[param] = params[param]
            else:
                for innerParam in param:
                    self.params[param].append(innerParam)

    def addInputPort(self, name=None, outputPort=None):
        if name is None:
            name = self.name + "_IN"+ str(len(self.inputPorts))
        iport = IPort(name, self, len(self.inputPorts))
        self.inputPorts.append(iport)

        if not outputPort is None:
            iport.connect(outputPort)
        return iport


    def generateSPLOperator(self):
        _op = {}
        _op["name"] = self.name

        _op["kind"] = self.kind
        _op["partitioned"] = False

        _outputs = []
        _inputs = []

        for output in self.outputPorts:
            _outputs.append(output.getSPLOutputPort())

        for input in self.inputPorts:
            _inputs.append(input.getSPLInputPort())
        _op["outputs"] = _outputs
        _op["inputs"] = _inputs
        _op["config"] = {}

        _params = {}
        for param in self.params:
            _value = {}
            _value["value"] = self.params[param]
            _params[param] = _value

        _op["parameters"] = _params

        return _op

    def _addOperatorFunction(self, function):
        if (function == None):
            return None
        if not hasattr(function, "__call__"):
            raise "argument to _addOperatorFunction is not callable"<commit_msg>Add mechanism for operator invocation parameters to have arbitrary JSON<commit_after># Licensed Materials - Property of IBM
# Copyright IBM Corp. 2015

import uuid
import json
import inspect
import pickle
import base64
import streamsx.topology.dependency
from streamsx.topology.schema import CommonSchema

class SPLGraph(object):

    def __init__(self, name=None):
        if name is None:
            name = str(uuid.uuid1()).replace("-", "")
        self.name = name
        self.operators = []
        self.resolver = streamsx.topology.dependency.DependencyResolver()

    def addOperator(self, kind, function=None, name=None):
        if name is None:
            if function is not None:
               if hasattr(function, '__name__'):
                   name = function.__name__ + "_" + str(len(self.operators))
               elif hasattr(function, '__class__'):
                   name = function.__class__.__name__ + "_" + str(len(self.operators))
            else:
               name = self.name + "_OP"+str(len(self.operators))
        if(kind.startswith("$")):    
            op = Marker(len(self.operators), kind, name, {}, self)                           
        else:
            op = SPLInvocation(len(self.operators), kind, function, name, {}, self)
        self.operators.append(op)
        if not function is None:
            if not inspect.isbuiltin(function):
                self.resolver.add_dependencies(inspect.getmodule(function))
        return op
    
    def addPassThruOperator(self):
        name = self.name + "_OP"+str(len(self.operators))
        op = SPLInvocation(len(self.operators), "com.ibm.streamsx.topology.functional.python::PyFunctionPassThru", None, name, {}, self)
        self.operators.append(op)
        return op

    def generateSPLGraph(self):
        _graph = {}
        _graph["name"] = self.name
        _graph["namespace"] = self.name
        _graph["public"] = True
        _graph["config"] = {}
        _graph["config"]["includes"] = []
        _ops = []
        self.addModules(_graph["config"]["includes"])
        self.addPackages(_graph["config"]["includes"])
        for op in self.operators:
            _ops.append(op.generateSPLOperator())

        _graph["operators"] = _ops
        return _graph
   
    def addPackages(self, includes):
        for package_path in self.resolver.packages:
           mf = {}
           mf["source"] = package_path
           mf["target"] = "opt/python/packages"
           includes.append(mf)

    def addModules(self, includes):
        for module_path in self.resolver.modules:
           mf = {}
           mf["source"] = module_path
           mf["target"] = "opt/python/modules"
           includes.append(mf)
           
    def getLastOperator(self):
        return self.operators[len(self.operators) -1]      
        
    def printJSON(self):
      print(json.dumps(self.generateSPLGraph(), sort_keys=True, indent=4, separators=(',', ': ')))

class SPLInvocation(object):

    def __init__(self, index, kind, function, name, params, graph):
        self.index = index
        self.kind = kind
        self.function = function
        self.name = name
        self.params = {}
        self.setParameters(params)
        self._addOperatorFunction(self.function)
        self.graph = graph

        self.inputPorts = []
        self.outputPorts = []

    def addOutputPort(self, oWidth=None, name=None, inputPort=None, schema= CommonSchema.Python):
        if name is None:
            name = self.name + "_OUT"+str(len(self.outputPorts))
        oport = OPort(name, self, len(self.outputPorts), schema, oWidth)
        self.outputPorts.append(oport)

        if not inputPort is None:
            oport.connect(inputPort)
        return oport

    def setParameters(self, params):
        for param in params:
            self.params[param] = params[param]

    def appendParameters(self, params):
        for param in params:
            if self.params.get(param) is None:
                self.params[param] = params[param]
            else:
                for innerParam in param:
                    self.params[param].append(innerParam)

    def addInputPort(self, name=None, outputPort=None):
        if name is None:
            name = self.name + "_IN"+ str(len(self.inputPorts))
        iport = IPort(name, self, len(self.inputPorts))
        self.inputPorts.append(iport)

        if not outputPort is None:
            iport.connect(outputPort)
        return iport


    def generateSPLOperator(self):
        _op = {}
        _op["name"] = self.name

        _op["kind"] = self.kind
        _op["partitioned"] = False

        _outputs = []
        _inputs = []

        for output in self.outputPorts:
            _outputs.append(output.getSPLOutputPort())

        for input in self.inputPorts:
            _inputs.append(input.getSPLInputPort())
        _op["outputs"] = _outputs
        _op["inputs"] = _inputs
        _op["config"] = {}

        _params = {}
        # Add parameters as their string representation
        # unless they value has a spl_json() function,
        # then use that
        _params = {}
        for name in self.params:
            param = self.params[name]
            try:
                _params[name] = param.spl_json()
            except:
                _value = {}
                _value["value"] = param
                _params[name] = _value
        _op["parameters"] = _params
        return _op

    def _addOperatorFunction(self, function):
        if (function == None):
            return None
        if not hasattr(function, "__call__"):
            raise "argument to _addOperatorFunction is not callable"
{'fraction_non_alphanumeric': 0.11424426511923112, 'fraction_numerical': 0.0016320609302747304, 'mean_word_length': 2.083589600223651, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2655098', 'n_tokens_mistral': 3181, 'n_tokens_neox': 3103, 'n_words': 705}