<commit_before>#Bringing the line into a base format if the same line repeats by varied k value

from wordsegment import segment
from collections import Counter
file = open('../label_idiom.txt')
idiomsEx = file.readlines()
num_arr = ['one','two','three','four','five','six','seven','eight','nine']
arr_contain_numbers = []
arr_repeat = []
for line in idiomsEx:
    line = line.split()
    seg_line = segment(line[0])
    space_line = " ".join(seg_line)
    if(any(char.isdigit() for char in space_line)):
        line_nonum = ''.join([i for i in space_line if not i.isdigit()])
        arr_contain_numbers.append(line_nonum)

for i in xrange(len(arr_contain_numbers)):
    k = arr_contain_numbers.count(arr_contain_numbers[i])
    if(k>1):arr_repeat.append(arr_contain_numbers[i])

l = Counter(arr_repeat)
for x in l:
    print x,l[x]<commit_msg>added comments<commit_after>#Bringing the line into a base format if the same line repeats by varied k value

from wordsegment import segment
from collections import Counter
file = open('../label_idiom.txt')
idiomsEx = file.readlines()
#num_arr = ['one','two','three','four','five','six','seven','eight','nine']
arr_contain_numbers = []
arr_repeat = []
for line in idiomsEx:
    line = line.split()
    seg_line = segment(line[0])
    space_line = " ".join(seg_line)
    #checks if any integer is present in the line
    if(any(char.isdigit() for char in space_line)):
        #eliminating the integers
        line_nonum = ''.join([i for i in space_line if not i.isdigit()])
        arr_contain_numbers.append(line_nonum)
#storing the lines which repeat
for i in xrange(len(arr_contain_numbers)):
    k = arr_contain_numbers.count(arr_contain_numbers[i])
    if(k>1):arr_repeat.append(arr_contain_numbers[i])
#counting the line repeated
l = Counter(arr_repeat)
for x in l:
    print x,l[x]
{'fraction_non_alphanumeric': 0.13017429193899782, 'fraction_numerical': 0.002178649237472767, 'mean_word_length': 4.687306501547988, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3192723', 'n_tokens_mistral': 641, 'n_tokens_neox': 616, 'n_words': 165}