<commit_before># Licensed Materials - Property of IBM
# Copyright IBM Corp. 2016
import os
import pickle

try:
    import dill
except ImportError:
    dill = pickle

import base64
import sys
import json

def __splpy_addDirToPath(dir):
    if os.path.isdir(dir):
        if dir not in sys.path:
            #print ("Adding dir to sys.path", dir)
            sys.path.append(dir)
                
def setupOperator(dir):
    pydir = os.path.join(dir, 'opt', 'python')
    __splpy_addDirToPath(os.path.join(pydir, 'modules'))
    __splpy_addDirToPath(os.path.join(pydir, 'packages'))
    #print("sys.path", sys.path)

def pickleReturn(function) :
    def _pickleReturn(v):
        return pickle.dumps(function(v))
    return _pickleReturn

# Given a callable 'callable', return a function
# that depickles the input and then calls 'callable'
# returning the callable's return
# The returned function must not maintain a reference
# to the passed in value as it will be a memory view
# object with memory that will become invalid after the call.
def pickle_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(pickle.loads(v))
    return _wf

# Given a callable 'callable', return a function
# that loads an object from the serialized JSON input
# and then calls 'callable' returning the callable's return
def json_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(json.loads(v))
    return _wf

def string_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(v)
    return _wf


# Given a callable 'callable', return a function
# that calls 'callable' with a python dictionary object 
# form of an spltuple returning the callable's return
def dict_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(v)
    return _wf

# Get the callable from the value
# passed into the SPL PyFunction operator.
#
# It is either something that is callable
# and is used directly or is string
# that is a encoded pickled class instance
#
# TODO throw exception
def _getCallable(f):
    if callable(f):
        return f
    if isinstance(f, str):
        ci = dill.loads(base64.b64decode(f))
        if callable(ci):
            return ci
    return None

##
## Set of functions that wrap the application's Python callable
## with a function that correctly handles the input and output
## (return) value. The input is from the SPL operator, i.e.
## a value obtained from a tuple attribute as a Python object.
## The output is the value (as a Python object) to be returned
## to the SPL operator to be set as a tuple attribute.
##
## The style is one of:
##
## pickle - Object is a Python byte string representing a picked object.
##          The object is depicked/pickled before being passed to/return from
##          the application function.
##
## json - Object is a Python unicode string representing a serialized
##          Json object. The object is deserialized/serialized before<commit_msg>Handle strings correctly in Python 2<commit_after># Licensed Materials - Property of IBM
# Copyright IBM Corp. 2016
import os
import pickle
from past.builtins import basestring

try:
    import dill
except ImportError:
    dill = pickle

import base64
import sys
import json

def __splpy_addDirToPath(dir):
    if os.path.isdir(dir):
        if dir not in sys.path:
            #print ("Adding dir to sys.path", dir)
            sys.path.append(dir)
                
def setupOperator(dir):
    pydir = os.path.join(dir, 'opt', 'python')
    __splpy_addDirToPath(os.path.join(pydir, 'modules'))
    __splpy_addDirToPath(os.path.join(pydir, 'packages'))
    #print("sys.path", sys.path)

def pickleReturn(function) :
    def _pickleReturn(v):
        return pickle.dumps(function(v))
    return _pickleReturn

# Given a callable 'callable', return a function
# that depickles the input and then calls 'callable'
# returning the callable's return
# The returned function must not maintain a reference
# to the passed in value as it will be a memory view
# object with memory that will become invalid after the call.
def pickle_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(pickle.loads(v))
    return _wf

# Given a callable 'callable', return a function
# that loads an object from the serialized JSON input
# and then calls 'callable' returning the callable's return
def json_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(json.loads(v))
    return _wf

def string_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(v)
    return _wf


# Given a callable 'callable', return a function
# that calls 'callable' with a python dictionary object 
# form of an spltuple returning the callable's return
def dict_in(callable) :
    ac = _getCallable(callable)
    def _wf(v):
        return ac(v)
    return _wf

# Get the callable from the value
# passed into the SPL PyFunction operator.
#
# It is either something that is callable
# and is used directly or is string
# that is a encoded pickled class instance
#
# TODO throw exception
def _getCallable(f):
    if callable(f):
        return f
    if isinstance(f, basestring):
        ci = dill.loads(base64.b64decode(f))
        if callable(ci):
            return ci
    return None

##
## Set of functions that wrap the application's Python callable
## with a function that correctly handles the input and output
## (return) value. The input is from the SPL operator, i.e.
## a value obtained from a tuple attribute as a Python object.
## The output is the value (as a Python object) to be returned
## to the SPL operator to be set as a tuple attribute.
##
## The style is one of:
##
## pickle - Object is a Python byte string representing a picked object.
##          The object is depicked/pickled before being passed to/return from
##          the application function.
##
## json - Object is a Python unicode string representing a serialized
##          Json object. The object is deserialized/serialized before
{'fraction_non_alphanumeric': 0.08073303615651313, 'fraction_numerical': 0.003467062902426944, 'mean_word_length': 3.2512280701754386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '18851', 'n_tokens_mistral': 1831, 'n_tokens_neox': 1787, 'n_words': 728}