<commit_before>import string

def num_there(s):
    return any(i.isdigit() for i in s)

	
def keep_bound(value, bound):
    return min(max(value, bound[0]), bound[1])


def convert_to_word_lst(sentence, lower=True):
    sentence = filter(lambda x: x in string.printable, sentence)
    exclude = '''!"#$%&\'()*+,:;<=>?@[\\]^_`{|}~-/\t''' + '\n'
    for e in exclude:
        sentence = sentence.replace(e, ' ')
    if lower:
        sentence = sentence.lower()
    word_sq = sentence.split(' ')
    ret = []
    for ind, w in enumerate(word_sq):
        if '.' in w:
            if num_there(w):
                try:  # detect whether there is a float number
                    float_word = float(w)
                    if w[len(w) - 1] == '.':
                        w.replace('.', '')
                    ret.append(w)
                except ValueError:
                    if w[len(w) - 1] == '.':
                        w.replace('.', '')
                    ret.append(w)
            else:
                ret.extend(w.split('.'))
        else:
            ret.append(w)
    return filter(lambda x: x.strip(), ret)
<commit_msg>add tool: wrap_star_digger()<commit_after>import string

def num_there(s):
    return any(i.isdigit() for i in s)

	
def keep_bound(value, bound):
    return min(max(value, bound[0]), bound[1])


def convert_to_word_lst(sentence, lower=True):
    sentence = filter(lambda x: x in string.printable, sentence)
    exclude = '''!"#$%&\'()*+,:;<=>?@[\\]^_`{|}~-/\t''' + '\n'
    for e in exclude:
        sentence = sentence.replace(e, ' ')
    if lower:
        sentence = sentence.lower()
    word_sq = sentence.split(' ')
    ret = []
    for ind, w in enumerate(word_sq):
        if '.' in w:
            if num_there(w):
                try:  # detect whether there is a float number
                    float_word = float(w)
                    if w[len(w) - 1] == '.':
                        w.replace('.', '')
                    ret.append(w)
                except ValueError:
                    if w[len(w) - 1] == '.':
                        w.replace('.', '')
                    ret.append(w)
            else:
                ret.extend(w.split('.'))
        else:
            ret.append(w)
    return filter(lambda x: x.strip(), ret)


def wrap_star_digger(item, type_str):
    """
    code used to extract data from Bing's wrap star
    :param item: wrap star obj
    :param type_str: target type string
    :return: list of all matched target, arranged in occurance
    """
    ret = []
    if type(item) == dict:
        if 'Type' in item and item['Type'] == type_str:  # 'Business.Consumer_Product.Description'
            if len(item['Value']) > 1:
                print 'length error!!!!!!!!!!!'
            return item['Value']
        else:
            for k in item:
                sub_ret = json_digger(item[k], type_str)
                if sub_ret:
                    ret.extend(sub_ret)
    elif type(item) == list:
        for i in item:
            sub_ret = json_digger(i, type_str)
            if sub_ret:
                ret.extend(sub_ret)
    return ret
{'fraction_non_alphanumeric': 0.14921549791866795, 'fraction_numerical': 0.002881844380403458, 'mean_word_length': 1.6142259414225941, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5535145', 'n_tokens_mistral': 997, 'n_tokens_neox': 943, 'n_words': 247}