<jupyter_start><jupyter_text>Preprocessing<jupyter_code>data = pd.read_csv('../Data/unpreprocessed_dataset.csv')
data['comment_len_by_words'] = data['body'].apply(lambda t: len(hazm.word_tokenize(t)))
min_max_len = data["comment_len_by_words"].min(), data["comment_len_by_words"].max()
print(f'Min: {min_max_len[0]} \tMax: {min_max_len[1]}')<jupyter_output>Min: 0 	Max: 27462<jupyter_text>Finding normal number of words in text<jupyter_code>def data_gl_than(data, less_than=100.0, greater_than=0.0, col='comment_len_by_words'):
    data_length = data[col].values

    data_glt = sum([1 for length in data_length if greater_than < length <= less_than])

    data_glt_rate = (data_glt / len(data_length)) * 100

    print(f'Texts with word length of greater than {greater_than} and less than {less_than} includes {data_glt_rate:.2f}% of the whole! means {data_glt}')
for i in range(15):
  data_gl_than(data, 100*i, 0)
for i in range(15):
  data_gl_than(data, 600, 50*i)
min_lenght , max_lenght = 50 , 600
# remove if less than min length or more than max length
data['comment_len_by_words'] = data['comment_len_by_words'].apply(lambda t_length: t_length if max_lenght>t_length>min_lenght else None)
print('lengh of data before dropout', str(len(data)))
data = data.dropna(subset=['comment_len_by_words'])
data = data.reset_index(drop=True)
print('lengh of data after dropout', str(len(data)))
fig = go.Figure()

fig.add_trace(go.Histogram(
    x=data['comment_len_by_words']
))

fig.update_layout(
    title_text='Distribution of word counts within comments',
    xaxis_title_text='Word Count',
    yaxis_title_text='Frequency',
    bargap=0.2,
    bargroupgap=0.2)

fig.show()
# import nltk
# from nltk.corpus import stopwords
# nltk.download()

def cleaning(text):
    text = text.strip()
    
    # regular cleaning
    text = clean(text)

    
    # normalizing
    normalizer = hazm.Normalizer()
    text = normalizer.normalize(text)
    
    # removing wierd patterns
    wierd_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\U00002702-\U000027B0"
        u"\U000024C2-\U0001F251"
        u"\U0001f926-\U0001f937"
        u'\U00010000-\U0010ffff'
        u"\u200d"
        u"\u2640-\u2642"
        u"\u2600-\u2B55"
        u"\u23cf"
        u"\u23e9"
        u"\u231a"
        u"\u3030"
        u"\ufe0f"
        u"\u2069"
        u"\u2066"
        # u"\u200c"
        u"\u2068"
        u"\u2067"
        "]+", flags=re.UNICODE)
    
    text = wierd_pattern.sub(r'', text)
    
    # removing extra spaces, hashtags
    text = re.sub("#", "", text)
    text = re.sub("\s+", " ", text)
    
    return text


data['cleaned_body'] = data['body'].apply(cleaning)
data['cleaned_body_len_by_words'] = data['cleaned_body'].apply(lambda t: len(hazm.word_tokenize(t)))
data = data.dropna(subset=['cleaned_body'])
# remove if less than min length or more than max length
data['cleaned_body_len_by_words'] = data['cleaned_body_len_by_words'].apply(lambda t_length: t_length if max_lenght>t_length>min_lenght else None)
print('lengh of data before dropout', str(len(data)))
data = data.dropna(subset=['cleaned_body_len_by_words'])
data = data.reset_index(drop=True)
print('lengh of data after dropout', str(len(data)))
data.head()<jupyter_output><empty_output><jupyter_text>Handling unbalanced data<jupyter_code># counter
count_data = data.copy()
count_data = count_data.dropna().reset_index(drop=True)
count = {}
tags_list = pd.read_excel('../Tags.xlsx',header = None)
tags_list.columns = ['tag']
for _,tag in tags_list.iterrows():
  if tag['tag']!='ویرگول' and tag['tag']!=None and tag['tag']!='':
    count[re.sub(' ','',tag['tag'].lower())] = 0

for indx,d in count_data.iterrows():
  tags = re.sub('\'|\[|\]','',d['tag']).split(',')
  
  for tag in tags:
    if tag.strip().lower() in tags_list.values:
      count[tag.strip().lower().replace(' ','')]+=1 
# count
def count_gl_than(count, less_than=100.0, greater_than=0.0):
    sum_count = sum([count[k] for k in list(count.keys())])

    count_glt = sum([count[k] for k in list(count.keys()) if greater_than < count[k] <= less_than])

    count_glt_rate = (count_glt / sum_count) * 100

    print(f'Tags with count of greater than {greater_than} and less than {less_than} includes {count_glt_rate:.2f}% of the whole! means {count_glt}')
for i in range(20):
  count_gl_than(count, 50*i, 0)
import  matplotlib.pyplot as plt
plt.hist([count[k] for k in list(count.keys())])
limit_number = 500
for i in range(len(data)):

    text=data.loc[i,"body"]
    tags = data.loc[i,'tag']
    data.loc[i,'tag'] = re.sub('ویرگول|c\+\+','',tags.lower())
    tags = re.sub('\[|\]|\'|=|ویرگول',' ',tags)
    tags_list = [t.strip() for t  in tags.split(',') if (t!=' ' and t!='' and (re.sub('[ ]*','',t) in list(count.keys())))]

    # balancing
    if len(tags_list) <1:
      data.loc[i,"body"]=None
      continue
    min_count = min([count[tag.replace(' ','')] for tag in tags_list if (tag.replace(' ','') in list(count.keys()))])

    if min_count<limit_number :

      replications = (limit_number - min_count)//min_count

      d_temp = data.iloc[i,:].copy()
      for _ in range(replications):

        data = data.append(d_temp)
        for tag in tags_list:
          count[tag.replace(' ','')] += 1
    
    for tag in tags_list:
        if count[tag.replace(' ','')]>limit_number:
          for t in tags_list:
            count[t.replace(' ','')] -=1
          data.loc[i,"body"]=None

    if i%500==0:
      print("Record: up to "+str(i)+" from " ,str(len(data)),"is processed.")
data = data.dropna().reset_index(drop=True)
data.head()
data.loc[:,['body','tag']]

data.loc[:,['body','tag']].to_csv("../Data/ParsBert_1_no_limitation.csv", index=False)
# counter
count_data = data.copy()
count_data = count_data.dropna().reset_index(drop=True)
count = {}
tags_list = pd.read_excel('../Tags.xlsx',header = None)
tags_list.columns = ['tag']
for _,tag in tags_list.iterrows():
  if tag['tag']!='ویرگول' and tag['tag']!=None and tag['tag']!='':
    count[re.sub(' ','',tag['tag'].lower())] = 0

for indx,d in count_data.iterrows():
  tags = re.sub('\'|\[|\]','',d['tag']).split(',')
  
  for tag in tags:
    if tag.strip().lower() in tags_list.values:
      count[tag.strip().lower().replace(' ','')]+=1 
count
plt.hist([count[k] for k in list(count.keys())])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17748851454823888, 'fraction_numerical': 0.03430321592649311, 'mean_word_length': 3.9477272727272728, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '528567', 'license': 'MIT', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2475, 'n_words': 484, 'path': 'FineTuning/ipython(guide)/ParsBert_preprocessing.ipynb', 'repo_name': 'ZeinabTaghavi/text_multi_label_classification', 'size': '6530'}