<commit_before># author: <NAME>
from hecuba.dict import *
from conf.apppath import apppath
import inspect
from pprint import pprint

def hecuba_filter(function, iterable):
    print "iterable:", iterable
    print iterable[0].__class__.__name__
    print iterable[1].__class__.__name__
    pprint (vars(iterable[0]))
    if hasattr(iterable[0], 'indexed'):
        print "indexed object"
        inspectedfunction = inspect.getsource(function)
        iterable[1].indexArguments = str(str(str(inspectedfunction).split(":")[1]).split(",")[0]).split(' and ')  # Args list
        print "iterable.indexArguments:", iterable[1].indexArguments
        return iterable[1]
    else:
        print "normal object"
        filtered = python_filter(function, iterable)
        return filtered

path = apppath + '/conf/storage_params.txt'

file = open(path, 'r')

for line in file:
    exec line

if not filter == hecuba_filter:
    python_filter = filter
    filter = hecuba_filter<commit_msg>really? still here?<commit_after># author: <NAME>
from hecuba.dict import *
from hecuba.storageobj import *
from conf.apppath import apppath
import inspect
from pprint import pprint

def hecuba_filter(function, iterable):
    print "iterable:", iterable
    print iterable[0].__class__.__name__
    print iterable[1].__class__.__name__
    pprint (vars(iterable[0]))
    print "issubclass(self, hecuba.StorageObj):", issubclass(self, hecuba.StorageObj)
    print "issubclass(self, hecuba.dict):      ", issubclass(self, hecuba.dict)
    if hasattr(iterable[0], 'indexed'):
        print "indexed object"
        inspectedfunction = inspect.getsource(function)
        iterable[1].indexArguments = str(str(str(inspectedfunction).split(":")[1]).split(",")[0]).split(' and ')  # Args list
        print "iterable.indexArguments:", iterable[1].indexArguments
        return iterable[1]
    else:
        print "normal object"
        filtered = python_filter(function, iterable)
        return filtered

path = apppath + '/conf/storage_params.txt'

file = open(path, 'r')

for line in file:
    exec line

if not filter == hecuba_filter:
    python_filter = filter
    filter = hecuba_filter
{'fraction_non_alphanumeric': 0.11797235023041475, 'fraction_numerical': 0.008294930875576038, 'mean_word_length': 3.8030973451327434, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2838322', 'n_tokens_mistral': 681, 'n_tokens_neox': 681, 'n_words': 164}