<commit_before>import re

def MUSE_v3_create_integers(input_dict):
    intStr = input_dict['intStr']
    intList = []
    for i in re.findall(r'\w+', intStr):
        try:
            intList.append(int(i))
        except:
            pass
    if input_dict['sort'].lower() == "true":
        intList.sort()
    return {'intList':intList}

def MUSE_v3_sum_integers(input_dict):
    intList = input_dict['intList']
    return {'sum':sum(intList)}

def MUSE_v3_pre_filter_integers(input_dict):
    return input_dict

def MUSE_v3_post_filter_integers(postdata,input_dict,output_dict):
    intListOut = postdata['intListOut']
    intList = []
    for i in intListOut:
        try:
            intList.append(int(i))
        except:
            pass
    return {'intList': intList}

def MUSE_v3_pre_display_summation(input_dict):
    return {}<commit_msg>new MUSE v3 widget(s)<commit_after>import re
import subprocess
import logging
try:
    from ladon.clients.jsonwsp import JSONWSPClient
except ImportError:
    logging.warning('Ladon package not available, most of MUSE stuff will not work.')



def MUSE_preprocessing(input_dict):
    url = input_dict['url']
    text = input_dict['text']

    import socket
    socket.setdefaulttimeout(None)

    cli = JSONWSPClient(url)
    result = cli.preprocessing(text=text)
    tokens = result.response_dict['result']
    return {'tokens': tokens}
#end

{'fraction_non_alphanumeric': 0.10839913854989232, 'fraction_numerical': 0.004307250538406317, 'mean_word_length': 3.4394904458598727, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5975141', 'n_tokens_mistral': 472, 'n_tokens_neox': 465, 'n_words': 97}