<jupyter_start><jupyter_text>-----------------   Union all csv files from data folder ---------------------------<jupyter_code>import os
import pandas as pd
import re
import nltk
import numpy as np
# Insert datapath
datapath = os.getcwd() +'/data'
#List wanted files to combine, now just csv files
files = [file for file in os.listdir(datapath) if file.endswith('.csv')]
#Lets make datafame from the first file
data = pd.read_csv(datapath+'/{}'.format(files[0]))
#lets union all csv files and then save file as all.csv
for i in range(1,len(files)):
    data = data.append(pd.read_csv(datapath+'/{}'.format(files[i])))
data.to_csv('all.csv')
len(data)<jupyter_output><empty_output><jupyter_text>----------------- TRANSFORMATIONS FOR DIFFERENT NEEDS ------------------------<jupyter_code>#### lets preprocess all.csv to make it nicer
df = pd.read_csv('all.csv')

#combine title and text
df['text_and_title'] = df['structs.text_title'] + "." + df['word']

# make label fasttext format
df['label'] = '__label__'+df['structs.text_sect']

#get rid of not needed columns
df_final = df[['label','text_and_title']]

# remove punctuation
df_final['punctuation_removed'] = df_final['text_and_title'].astype(str).apply(lambda row: re.sub(r'[^\w\s]',' ',row))

#remove numbers
df_final['text_and_title_non_number'] = df_final['text_and_title'].str.replace('\d+', '')

#lowercase
df_final['lowered'] = df_final['punctuation_removed'].apply(lambda row: str.lower(row))

#tokenize
df_final['tokenized'] = df_final['lowered'].apply(lambda r: nltk.word_tokenize(r))

# get char and word counts
df_final['char_count'] = df_final.lowered.replace(' ','').map(len)
df_final['word_count'] = df_final.tokenized.map(len)

# Save this to csv
df_final.to_csv('all_preprocessed_no_filtering.csv')

df_ft = df[['label','text_and_title']]
# Save in the right format for FastText training
df_ft.to_csv('fasttext_embeddings_train.txt', sep='\t', index = False, header = False)
df_ft.iloc[0:int(len(df)*0.8)].to_csv('train.txt', sep='\t', index = False, header = False)
df_ft.iloc[int(len(df)*0.8):int(len(df)*0.9)].to_csv('test.txt', sep='\t', index = False, header = False)<jupyter_output>F:\Anaconda3\envs\FinBertTrainEnv\lib\site-packages\ipykernel_launcher.py:14: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
F:\Anaconda3\envs\FinBertTrainEnv\lib\site-packages\ipykernel_launcher.py:17: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
F:\Anaconda3\envs\FinBertTrainEnv\lib\site-packages\ipykernel_launcher.py:20: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: ht[...]<jupyter_text>---------------------------- Filtering for BERT model ---------------------------------<jupyter_code>#function to cut dataframe by the amount of words in message
def cut_df_by_word_count(df,min_words, max_words):
    df = df[df.word_count.between(min_words,max_words)]
    return df

def cut_df_by_topic_counts(df, column, min_occurances, max_occurances):
    value_counts = test_df['{}'.format(column)].value_counts()
    a = value_counts.between(min_occurances,max_occurances)
    b = [a.index[i] for i,x in enumerate(a.index) if a[i]]
    df = df[df['{}'.format(column)].isin(b)]
    return df
len(df_final)
# cut test_df by word_count eg.
# here only take messages that have words between 50 to 500
# test_df = cut_df_by_word_count(messages, 50, 500)
lower_word_limit = 50
upper_word_limit = 500

test_df = cut_df_by_word_count(df_final, 50, 500)
## if you want you can also cut df by the amount of samples in topic
## here for example we cut topics keep only topics that have between 2000 to 20 000 messages
#test_df = cut_df_by_topic_counts(test_df, 'label',2000, 20000)
#lower_topic_count = 2000
#upper_topic_count = 20000
#test_df = cut_df_by_topic_counts(test_df, 'label',lower_topic_count, upper_topic_count)
test_df.to_csv('all_preprocessed_filtered.csv')
test_df['label'].value_counts()
len(test_df)
pd.value_counts(test_df['label']).plot.barh()
test_df.word_count.hist(bins=np.linspace(0,1000,100))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16089216110870508, 'fraction_numerical': 0.01580770896491988, 'mean_word_length': 4.998701298701299, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '154271', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1632, 'n_words': 381, 'path': 'union_csv_files_and_preprocess.ipynb', 'repo_name': 'zhangby2085/FinBERT_tutorial', 'size': '4618'}