<commit_before><commit_msg>push LoadUtil.py<commit_after>
def rmload_start_end(el, field, startstr, endstr):
    value = el.get_output_value(field)
    if value is not None:
        spos = value.find(startstr)
        if spos != -1:
            epos = value.find(endstr, spos+len(startstr))
            if epos != -1:
                newvalue = value[:spos] + value[epos + len(endstr):]
                el.replace_value(field, newvalue)
    return el

def rmload_string(el, field, str):
    value = el.get_output_value(field)
    if value is not None:
        pos = value.find(str)
        if pos != -1:
            newvalue = value[:pos] + value[pos+len(str):]
            el.replace_value(field, newvalue)
    return el
        


{'fraction_non_alphanumeric': 0.11748633879781421, 'fraction_numerical': 0.004098360655737705, 'mean_word_length': 2.2149122807017543, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4025404', 'n_tokens_mistral': 246, 'n_tokens_neox': 244, 'n_words': 56}