<commit_before># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2016
import os
import pickle
from past.builtins import basestring

import streamsx.ec as ec

try:
    import dill
    dill.settings['recurse'] = True
except ImportError:
    dill = pickle

import base64
import sys
import json
from pkgutil import extend_path
import streamsx

def __splpy_addDirToPath(dir):
    if os.path.isdir(dir):
        if dir not in sys.path:
            sys.path.insert(0, dir)
            # In case a streamsx module (e.g. streamsx.bm) 
            # is included in the additional code<commit_msg>Fix cloudpickle bug where it breaks dill.<commit_after># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2016
import os
import sys
import pickle
from past.builtins import basestring

import streamsx.ec as ec

try:
    import dill
    if sys.version_info.major == 3:
        dill.dill._reverse_typemap['ClassType'] = type
    dill.settings['recurse'] = True
except ImportError:
    dill = pickle

import base64
import json
from pkgutil import extend_path
import streamsx

def __splpy_addDirToPath(dir):
    if os.path.isdir(dir):
        if dir not in sys.path:
            sys.path.insert(0, dir)
            # In case a streamsx module (e.g. streamsx.bm) 
            # is included in the additional code
{'fraction_non_alphanumeric': 0.07371556217423679, 'fraction_numerical': 0.012658227848101266, 'mean_word_length': 2.9881305637982196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5453745', 'n_tokens_mistral': 437, 'n_tokens_neox': 417, 'n_words': 146}