<commit_before># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2015,2016

from __future__ import unicode_literals
from future.builtins import *

import sys
import uuid
import json
import inspect
import pickle
from enum import Enum

try:
    import dill
    dill.settings['recurse'] = True
except ImportError:
    dill = pickle

import types
import base64
import re
import streamsx.topology.dependency
import streamsx.topology.functions
import streamsx.topology.param
import streamsx.spl.op
from streamsx.topology.schema import CommonSchema, StreamSchema
from streamsx.topology.schema import _stream_schema


def _fix_namespace(ns):
    ns = str(ns)
    sns = ns.split('.')
    if len(sns) == 1:
        return re.sub(r'\W+', '', ns, flags=re.UNICODE)
    for i in range(0,len(sns)):
        sns[i] = re.sub(r'\W+', '', sns[i], flags=re.UNICODE)

    for i in range(len(sns), 0):
        if len(sns[i]) == 0:
            sns.pop(i)

    return '.'.join(sns)

def _as_spl_expr(value):
    """ Return value converted to an SPL expression if
    needed other otherwise value.
    """
    if hasattr(value, 'spl_json'):
        return value
      
    if isinstance(value, Enum):
        value = streamsx.spl.op.Expression.expression(value.name)
    return value

class SPLGraph(object):

    def __init__(self, topology, name=None, namespace=None):
        if name is None:
            name = str(uuid.uuid1()).replace("-", "")

        if namespace is None:
            namespace = name
        
        # Allows Topology or SPLGraph to be passed to submit
        self.graph = self
        # Remove 'awkward characters' from names
        self.name = re.sub(r'\W+', '', str(name), flags=re.UNICODE)
        self.namespace = _fix_namespace(namespace)
        self.topology = topology
        self.operators = []
        self.resolver = streamsx.topology.dependency._DependencyResolver(self.topology)
        self._views = []
        self._spl_toolkits = []
        self._used_names = {'list', 'tuple', 'int'}
        self._layout_group_id = 0
        self._colocate_tag_mapping = {}

    def get_views(self):
        return self._views

    def add_views(self, view):
        self._views.append(view)

    def _requested_name(self, name, action=None, func=None):
        """Create a unique name for an operator or a stream.
        """
        if name is not None:
            if name in self._used_names:
                # start at 2 for the "second" one of this name
                n = 2
                while True:
                    pn = name + '_' + str(n)
                    if pn not in self._used_names:
                        self._used_names.add(pn)
                        return pn
                    n += 1
            else:
                self._used_names.add(name)
                return name

        if func is not None:
            if hasattr(func, '__name__'):
                name = func.__name__
                if name == '<lambda>':
                    # Avoid use of <> characters in name
                    # as they are converted to unicode
                    # escapes in SPL identifier
                    name = action + '_lambda'
            elif hasattr(func, '__class__'):
                name = func.__class__.__name__

        if name is None:
            if action is not None:
                name = action
            else:
                name = self.name

        # Recurse once to get unique version of name
        return self._requested_name(name)


    def addOperator(self, kind, function=None, name=None, params=None, sl=None):
        if(params is None):
            params = {}

        if name is None:
            name = self._requested_name(None,action="Op", func = function)

        if(kind.startswith("$")):    
            op = Marker(len(self.operators), kind, name, {}, self)                           
        else:
            if function is not None:
                params['toolkitDir'] = streamsx.topology.param.toolkit_dir()
            op = _SPLInvocation(len(self.operators), kind, function, name, params, self, sl=sl)
        self.operators.append(op)
        if not function is None:
            dep_instance = function
            if isinstance(function, streamsx.topology.runtime._WrappedInstance):
                dep_instance = type(function._callable)

            if not inspect.isbuiltin(dep_instance):
                self.resolver.add_dependencies(inspect.getmodule(dep_instance))
        return op
    
    def addPassThruOperator(self):
        name = self.name + "_OP"+str(len(self.operators))
        op = _SPLInvocation(len(self.operators), "spl.relational::Functor", None, name, {}, self)
        self.operators.append(op)
        return op

    def _next_layout_group_id(self):
        lgi = '__spl_lg_' + str(self._layout_group_id)
        self._layout_group_id += 1
        return lgi

    def generateSPLGraph(self):
        self.topology._prepare()
        _graph = {}
        _graph["name"] = self.name
        _graph["namespace"] = self.namespace
        _graph["public"] = True
        _graph["config"] = {}
        _graph["config"]["includes"] = []
        _graph['config']['spl'] = {}
        _graph['config']['spl']['toolkits'] = self._spl_toolkits
        if self._colocate_tag_mapping:
            _graph['config']['colocateTagMapping'] = self._colocate_tag_mapping
        _ops = []
        self._add_modules(_graph["config"]["includes"])
        self._add_packages(_graph["config"]["includes"])
        self._add_files(_graph["config"]["includes"])
        for op in self.operators:
            _ops.append(op.generateSPLOperator())

        _graph["operators"] = _ops
        return _graph
   
    def _add_packages(self, includes):
        for package_path in self.resolver.packages:
           mf = {}
           mf["source"] = package_path
           mf["target"] = "opt/python/packages"
           includes.append(mf)

    def _add_modules(self, includes):
        for module_path in self.resolver.modules:
           mf = {}
           mf["source"] = module_path
           mf["target"] = "opt/python/modules"
           includes.append(mf)

    def _add_files(self, includes):
         fls = self.topology._files
         for location in fls:
             files = fls[location]
             for path in files:
                 if isinstance(path, str):
                     # Simple file with a source to copy
                     f = {}
                     f['source'] = path
                     f['target'] = location
                     includes.append(f)
                 else:
                     # Arbitray file description
                     includes.append(path)

    def getLastOperator(self):
        return self.operators[len(self.operators) -1]      
        
class _SPLInvocation(object):

    def __init__(self, index, kind, function, name, params, graph, view_configs = None, sl=None):
        self.index = index
        self.kind = kind
        self.function = function
        self.name = name
        self.category = None
        self.params = {}
        self.setParameters(params)
        self._addOperatorFunction(self.function)
        self.graph = graph
        self.viewable = True<commit_msg>Check for instance of str in a Python 2/3 compatible way<commit_after># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2015,2016

from __future__ import unicode_literals
from future.builtins import *
from past.builtins import basestring

import sys
import uuid
import json
import inspect
import pickle
from enum import Enum

try:
    import dill
    dill.settings['recurse'] = True
except ImportError:
    dill = pickle

import types
import base64
import re
import streamsx.topology.dependency
import streamsx.topology.functions
import streamsx.topology.param
import streamsx.spl.op
from streamsx.topology.schema import CommonSchema, StreamSchema
from streamsx.topology.schema import _stream_schema


def _fix_namespace(ns):
    ns = str(ns)
    sns = ns.split('.')
    if len(sns) == 1:
        return re.sub(r'\W+', '', ns, flags=re.UNICODE)
    for i in range(0,len(sns)):
        sns[i] = re.sub(r'\W+', '', sns[i], flags=re.UNICODE)

    for i in range(len(sns), 0):
        if len(sns[i]) == 0:
            sns.pop(i)

    return '.'.join(sns)

def _as_spl_expr(value):
    """ Return value converted to an SPL expression if
    needed other otherwise value.
    """
    if hasattr(value, 'spl_json'):
        return value
      
    if isinstance(value, Enum):
        value = streamsx.spl.op.Expression.expression(value.name)
    return value

class SPLGraph(object):

    def __init__(self, topology, name=None, namespace=None):
        if name is None:
            name = str(uuid.uuid1()).replace("-", "")

        if namespace is None:
            namespace = name
        
        # Allows Topology or SPLGraph to be passed to submit
        self.graph = self
        # Remove 'awkward characters' from names
        self.name = re.sub(r'\W+', '', str(name), flags=re.UNICODE)
        self.namespace = _fix_namespace(namespace)
        self.topology = topology
        self.operators = []
        self.resolver = streamsx.topology.dependency._DependencyResolver(self.topology)
        self._views = []
        self._spl_toolkits = []
        self._used_names = {'list', 'tuple', 'int'}
        self._layout_group_id = 0
        self._colocate_tag_mapping = {}

    def get_views(self):
        return self._views

    def add_views(self, view):
        self._views.append(view)

    def _requested_name(self, name, action=None, func=None):
        """Create a unique name for an operator or a stream.
        """
        if name is not None:
            if name in self._used_names:
                # start at 2 for the "second" one of this name
                n = 2
                while True:
                    pn = name + '_' + str(n)
                    if pn not in self._used_names:
                        self._used_names.add(pn)
                        return pn
                    n += 1
            else:
                self._used_names.add(name)
                return name

        if func is not None:
            if hasattr(func, '__name__'):
                name = func.__name__
                if name == '<lambda>':
                    # Avoid use of <> characters in name
                    # as they are converted to unicode
                    # escapes in SPL identifier
                    name = action + '_lambda'
            elif hasattr(func, '__class__'):
                name = func.__class__.__name__

        if name is None:
            if action is not None:
                name = action
            else:
                name = self.name

        # Recurse once to get unique version of name
        return self._requested_name(name)


    def addOperator(self, kind, function=None, name=None, params=None, sl=None):
        if(params is None):
            params = {}

        if name is None:
            name = self._requested_name(None,action="Op", func = function)

        if(kind.startswith("$")):    
            op = Marker(len(self.operators), kind, name, {}, self)                           
        else:
            if function is not None:
                params['toolkitDir'] = streamsx.topology.param.toolkit_dir()
            op = _SPLInvocation(len(self.operators), kind, function, name, params, self, sl=sl)
        self.operators.append(op)
        if not function is None:
            dep_instance = function
            if isinstance(function, streamsx.topology.runtime._WrappedInstance):
                dep_instance = type(function._callable)

            if not inspect.isbuiltin(dep_instance):
                self.resolver.add_dependencies(inspect.getmodule(dep_instance))
        return op
    
    def addPassThruOperator(self):
        name = self.name + "_OP"+str(len(self.operators))
        op = _SPLInvocation(len(self.operators), "spl.relational::Functor", None, name, {}, self)
        self.operators.append(op)
        return op

    def _next_layout_group_id(self):
        lgi = '__spl_lg_' + str(self._layout_group_id)
        self._layout_group_id += 1
        return lgi

    def generateSPLGraph(self):
        self.topology._prepare()
        _graph = {}
        _graph["name"] = self.name
        _graph["namespace"] = self.namespace
        _graph["public"] = True
        _graph["config"] = {}
        _graph["config"]["includes"] = []
        _graph['config']['spl'] = {}
        _graph['config']['spl']['toolkits'] = self._spl_toolkits
        if self._colocate_tag_mapping:
            _graph['config']['colocateTagMapping'] = self._colocate_tag_mapping
        _ops = []
        self._add_modules(_graph["config"]["includes"])
        self._add_packages(_graph["config"]["includes"])
        self._add_files(_graph["config"]["includes"])
        for op in self.operators:
            _ops.append(op.generateSPLOperator())

        _graph["operators"] = _ops
        return _graph
   
    def _add_packages(self, includes):
        for package_path in self.resolver.packages:
           mf = {}
           mf["source"] = package_path
           mf["target"] = "opt/python/packages"
           includes.append(mf)

    def _add_modules(self, includes):
        for module_path in self.resolver.modules:
           mf = {}
           mf["source"] = module_path
           mf["target"] = "opt/python/modules"
           includes.append(mf)

    def _add_files(self, includes):
         fls = self.topology._files
         for location in fls:
             files = fls[location]
             for path in files:
                 if isinstance(path, basestring):
                     # Simple file with a source to copy
                     f = {}
                     f['source'] = path
                     f['target'] = location
                     includes.append(f)
                 else:
                     # Arbitray file description
                     includes.append(path)

    def getLastOperator(self):
        return self.operators[len(self.operators) -1]      
        
class _SPLInvocation(object):

    def __init__(self, index, kind, function, name, params, graph, view_configs = None, sl=None):
        self.index = index
        self.kind = kind
        self.function = function
        self.name = name
        self.category = None
        self.params = {}
        self.setParameters(params)
        self._addOperatorFunction(self.function)
        self.graph = graph
        self.viewable = True
{'fraction_non_alphanumeric': 0.10477299185098952, 'fraction_numerical': 0.0031500376634938025, 'mean_word_length': 1.9132256134051466, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4231908', 'n_tokens_mistral': 4236, 'n_tokens_neox': 4074, 'n_words': 1039}