<jupyter_start><jupyter_text>Check if any spaces are marked in the toxic spans<jupyter_code>def get_space_count(df):
    df['space_count'] = df.apply(lambda x: np.sum([1 if x['text'][i]==' ' else 0 for i in eval(x['spans'])]),axis=1)
    return df['space_count'].sum(), df['space_count'].mean(),  df['space_count'].std()
print(['%.2f'% i for i in get_space_count(train)])
print(['%.2f'% i for i in get_space_count(trial)])
print(['%.2f'% i for i in get_space_count(test)])<jupyter_output>['575.00', '0.29', '3.19']<jupyter_text>Check if any words are cut across spans<jupyter_code>def check_if_word_cut(text,contiguous_spans):
    words_cuts = []
    for i in contiguous_spans:
        words_cut = 0
        if i[0]==0 and i[-1]==len(text)-1:
            words_cut = 0
        elif i[0]==0:
            if text[i[-1]]!=' ' and text[i[-1]+1].isalnum():
                words_cut+=1
        elif i[-1]==len(text)-1:
            if text[i[0]]!=' ' and text[i[0]-1].isalnum():
                words_cut +=1
        else:
            if text[i[0]]!=' ' and text[i[0]-1].isalnum():
                words_cut +=1
            if text[i[-1]]!=' ' and text[i[-1]+1].isalnum():
                words_cut +=1
        words_cuts.append(words_cut)
    return words_cuts     
def words_cut_count(df):
    df['contiguous_spans'] = df.apply(lambda x : get_continuous_spans(eval(x['spans'])),axis=1)
    df['words_cut'] = df.apply(lambda x: np.sum(check_if_word_cut(x['text'],x['contiguous_spans'])),axis=1)
    return df['words_cut'].sum(),df['words_cut'].mean(),df['words_cut'].std()
print(['%.2f'% i for i in words_cut_count(train)])
print(['%.2f'% i for i in words_cut_count(trial)])
print(['%.2f'% i for i in words_cut_count(test)])
def print_contiguous_spans(text,contiguous_spans):
    return [text[i[0]:i[-1]+1] for i in contiguous_spans]
# train[train['words_cut']>0].apply(lambda x: (x['text'],len(x['text']),x['spans'],x['contiguous_spans'],print_contiguous_spans(x['text'],x['contiguous_spans']),x['words_cut']),axis=1).values<jupyter_output><empty_output><jupyter_text>Check if any spans end or start with space<jupyter_code>import string
def check_if_start_or_end_with_whitespace(text,contiguous_spans):
    start_or_end_with_space = []
    for i in contiguous_spans:
        s_e = 0
        if text[i[0]] in string.whitespace:
            s_e+=1
        if text[i[-1]] in string.whitespace:
            s_e+=1
        start_or_end_with_space.append(s_e)
    return start_or_end_with_space 
def start_end_with_space(df):
    df['start_or_end_with_space'] = df.apply(lambda x: np.sum(check_if_start_or_end_with_whitespace(x['text'],x['contiguous_spans'])),axis=1)
    return df['start_or_end_with_space'].sum(),df['start_or_end_with_space'].mean(),df['start_or_end_with_space'].std()
    
print(['%.2f'% i for i in start_end_with_space(train)])
print(['%.2f'% i for i in start_end_with_space(trial)])
print(['%.2f'% i for i in start_end_with_space(test)])
# train[train['start_or_end_with_space']>0].apply(lambda x: (x['text'],len(x['text']),x['start_or_end_with_space'],print_contiguous_spans(x['text'],x['contiguous_spans'])),axis=1).values<jupyter_output><empty_output><jupyter_text>Clean out spans which are half in, remove trailing and beginning spaces<jupyter_code>def find_word_by_character_index(text, idx):
    if(text[idx].isalnum()):
        backward = idx
        forward = idx
        while(backward>-1 and text[backward].isalnum()):
            backward-=1
        while(forward<len(text) and text[forward].isalnum()):
            forward+=1

        return text[backward+1:forward], backward+1,forward-1
    else:
        return text[idx], idx, idx
train.loc[20,'spans']
train.loc[20,'text']
find_word_by_character_index(train.loc[20,'text'],22)
# def clean_text(text,contiguous_spans):
#     new_contiguous_spans = []

#     for i in contiguous_spans:
#         start = i[0] 
#         end = i[-1]

#         if start==0 and end==len(text)-1:
#             new_contiguous_spans.append([start,end])

#         elif start==0:
#             if text[end].isalnum() and text[end+1].isalnum():

#                 full_word,full_start,full_end = find_word_by_character_index(text,end)
#                 cut_word_len = end-full_start+1
#                 if(cut_word_len*2>=len(full_word)):
#                     new_contiguous_spans.append([start,full_end])
#                 else:
#                     new_contiguous_spans.append([start,full_start-1])


#         elif i[-1]==len(text)-1:
#             if text[start].isalnum() and text[start-1].isalnum():
#                 full_word, full_start,full_end = find_word_by_character_index(text,start)
#                 cut_word_len = full_end-start+1
#                 if(cut_word_len*2>=len(full_word)):
#                     new_contiguous_spans.append([full_start,end])
#                 else:
#                     new_contiguous_spans.append([full_end+1,end])
                
                
#         else:
#             new_start = start
#             new_end = end
           
#             if text[start].isalnum() and text[start-1].isalnum():
#                 full_word, full_start,full_end = find_word_by_character_index(text,start)
#                 cut_word_len = full_end-start+1
#                 if(cut_word_len*2>=len(full_word)):
#                     new_start = full_start
#                 else:
#                     new_start = full_end+1

#             if text[end].isalnum() and text[end+1].isalnum():
#                 full_word, full_start,full_end = find_word_by_character_index(text, end)
#                 cut_word_len = end-full_start+1
#                 if(cut_word_len*2>=len(full_word)):
#                     new_end = full_end
#                 else:
#                     new_end = full_start-1
#             new_contiguous_spans.append([new_start,new_end])
#     ## Remove Spaces from span beginning and end

#     newest_contiguous_spans = []
#     for i in new_contiguous_spans:
#         start = i[0]
#         end = i[-1]
#         while start<=end:
#             if(not (text[start].isalnum()) or not (text[end].isalnum())):
#                 if not (text[start].isalnum()):
#                     start+=1
#                 if not (text[end].isalnum()):
#                     end-=1
#             else:
#                 break
#         if(start<=end):
#             newest_contiguous_spans.append([start,end])
#     return newest_contiguous_spans

text = "Hello How are you sir, My name is <NAME>?"
contiguous_spans = get_continuous_spans([1,2,3,4,6,7,11,12,13,14,15,16,17, 18, 19, 22,23,24,25,30,31,32,33,34,35,36, 41,42,43,44])
print_contiguous_spans(text,contiguous_spans)
# new_contiguous_spans = clean_text(text,contiguous_spans)
#print_contiguous_spans(text,new_contiguous_spans)
# clean_train = new_train.copy()
# clean_train['contiguous_spans'] = clean_train.apply(lambda x: clean_text(x['text'],x['contiguous_spans']),axis=1)
# clean_trial = trial.copy()
# clean_trial['contiguous_spans'] = clean_trial.apply(lambda x:clean_text(x['text'],x['contiguous_spans']),axis=1)
#start_end_with_space(clean_train)
#start_end_with_space(clean_trial)
# def words_cut_count_clean(df):
#     df['words_cut'] = df.apply(lambda x: np.sum(check_if_word_cut(x['text'],x['contiguous_spans'])),axis=1)
#     return df['words_cut'].sum(),df['words_cut'].mean(),df['words_cut'].std()
# words_cut_count_clean(clean_train)
# words_cut_count_clean(clean_trial)
# clean_train.to_csv('../data/clean_train.csv',index=False)
# clean_trial.to_csv('../data/clean_trial.csv',index=False)
# clean_train = pd.read_csv('../data/clean_train.csv')
# clean_trial = pd.read_csv('../data/clean_trial.csv')
# def get_spans_from_contiguous(contiguous_spans):
#     spans = []
#     for i in eval(contiguous_spans):
#         spans+=list(range(i[0],i[-1]+1))
#     return spans
# clean_train['spans'] = clean_train['contiguous_spans'].apply(get_spans_from_contiguous)
# clean_trial['spans'] = clean_trial['contiguous_spans'].apply(get_spans_from_contiguous)
# clean_train[['spans','text']].to_csv('../data/clean_train.csv',index=False)
# clean_trial[['spans','text']].to_csv('../data/clean_trial.csv',index=False)
# new_train.to_csv('../data/modified_train.csv',index=False)<jupyter_output><empty_output><jupyter_text>Tokens Count<jupyter_code># clean_train = pd.read_csv('../data/clean_train.csv')
# clean_trial = pd.read_csv('../data/clean_trial.csv')
# reduced_train = pd.read_csv('../data/modified_train.csv')
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
def tokenize_text(df):
    df['#tokens'] = df.apply(lambda x: len(tokenizer.tokenize(x['text'])),axis=1)
    df['#words'] = df.apply(lambda x: len(x['text'].split()),axis=1)
    df['#chars'] = df.apply(lambda x: len(x['text']),axis=1)
    
lists = []
for df in [train,trial,test]:
    tokenize_text(df)
    lists.append(['%.2f' % i for i in (df['#tokens'].mean(),df['#tokens'].std(),df['#tokens'].max(),df['#tokens'].min())]+['%.2f' % i for i in (df['#words'].mean(),df['#words'].std(),df['#words'].max(),df['#words'].min())] +['%.2f' % i for i in (df['#chars'].mean(),df['#chars'].std(),df['#chars'].max(),df['#chars'].min())] )
print(pd.DataFrame(lists, columns=["#Tokens(mean)","#Tokens(std)","#Tokens(max)","#Tokens(min)","#Words(mean)","#Words(std)","#Words(max)","#Words(min)", "#Chars(mean)","#Chars(std)","#Chars(max)","#Chars(min)"]).T.to_markdown())<jupyter_output>|               |       0 |      1 |       2 |
|:--------------|--------:|-------:|--------:|
| #Tokens(mean) |   47.5  |  46.1  |   43.12 |
| #Tokens(std)  |   45.46 |  43.82 |   39.88 |
| #Tokens(max)  |  335    | 234    |  291    |
| #Tokens(min)  |    1    |   1    |    2    |
| #Words(mean)  |   35.95 |  35.01 |   32.86 |
| #Words(std)   |   34.97 |  34.42 |   31.01 |
| #Words(max)   |  192    | 182    |  186    |
| #Words(min)   |    1    |   1    |    1    |
| #Chars(mean)  |  204.57 | 199.47 |  186.41 |
| #Chars(std)   |  201.37 | 196.63 |  178.76 |
| #Chars(max)   | 1000    | 998    | 1000    |
| #Chars(min)   |    4    |   5    |    6    |
{'fraction_non_alphanumeric': 0.18091942659416707, 'fraction_numerical': 0.02847256549678695, 'mean_word_length': 3.2593684210526317, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '567484', 'license': 'MIT', 'n_tokens_mistral': 3927, 'n_tokens_neox': 3756, 'n_words': 526, 'path': 'notebooks/Exploratory Data Analysis & Preprocessing.ipynb', 'repo_name': 'gchhablani/toxic-spans-detection', 'size': '10125'}