<jupyter_start><jupyter_text>Config<jupyter_code>data_input_dir_str = "./Data/Raw/SageWrite_corpus_0.1_origin.csv"
data_output_dir_str = "./Data/Preprocessed/preprocessed.csv"

outline_columns_lst = ["outline_1", "outline_2", "outline_3"]

colnames_change_dict = {outline_columns_lst[0]: 'outline',
                   outline_columns_lst[1]: 'outline',
                   outline_columns_lst[2]: 'outline'}

name_replacement_dict = {1: 1, 2: 1, 3:1} 
data = pd.read_csv(data_input_dir_str)
data = split_data_add_type(data, "type")
data = concat_dfs(data, outline_columns_lst, colnames_change_dict)
data = data.replace(name_replacement_dict)
plot = data.drop(['sb_summary_id', 'original_text_id', 'original_text'], axis = 1).hist(figsize=(15, 12))
data = set_column_text_len(data, "outline")
data = set_column_text_len(data, "original_text")
plot_column_distribution(data, "outlineTokensLength")
plot_column_distribution(data, "original_textTokensLength")
data = data[data.outlineTokensLength>20]
len(data)
#data[0:1]
data.reset_index(drop=True).to_csv(data_output_dir_str, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13452914798206278, 'fraction_numerical': 0.02062780269058296, 'mean_word_length': 7.651162790697675, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '584485', 'license': 'MIT', 'n_tokens_mistral': 430, 'n_tokens_neox': 418, 'n_words': 55, 'path': '1.Preprocess.ipynb', 'repo_name': 'petervajdecka02947/SageWrite', 'size': '1115'}