<jupyter_start><jupyter_text>1.Clean data<jupyter_code>import os
import pandas as pd
import numpy as np

#path_host = r"C:\Users\client_2\Documents\\"

with open('books_new_N2.csv') as f:
    data = pd.read_csv(f)
data.drop_duplicates()

clean_author = data["Title"]
clean_subgenre = data["SubGenre"]

for i in range(len(clean_author)):   
    if ', ' in str(clean_author[i]):
        index = str(clean_author[i]).index(",")
        if 'The' in str(clean_author[i]):
            clean_author[i] = 'The ' + clean_author[i][:index]
        if 'A' in str(clean_author[i]):
            clean_author[i] = 'A ' + clean_author[i][:index]
        if 'An' in str(clean_author[i]):
            clean_author[i] = 'An ' + clean_author[i][:index]
        if 'Foudation' in str(clean_author[i]):
            clean_author[i] = 'Foudation ' + clean_author[i][:index]
            
for i in range(len(clean_subgenre)):
    if '_' in str(clean_subgenre[i]):
        clean_subgenre[i] = str(clean_subgenre[i]).replace("_"," ",1)

check_path = r"C:\Users\Admin\Desktop\Contain\\"
data.to_csv(check_path+"clean.csv",index=False)
os.listdir(check_path)<jupyter_output><empty_output><jupyter_text>2.Split data into 2 parts<jupyter_code>import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

data = pd.read_csv("clean.csv")
# split dataset using train_test_split
client_1, client_2 = train_test_split(data, train_size = 0.5, test_size = 0.5, random_state = 2021)
#save file
client_1.to_csv(check_path+"client_1.csv", index = False)
client_2.to_csv(check_path+"client_2.csv", index = False)
os.listdir(check_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14622356495468278, 'fraction_numerical': 0.012084592145015106, 'mean_word_length': 4.4295081967213115, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '622233', 'license': 'MIT', 'n_tokens_mistral': 634, 'n_tokens_neox': 615, 'n_words': 114, 'path': 'client/MarkNice_Step_2.ipynb', 'repo_name': 'nprm1243/-n-THNMCNTT-', 'size': '1655'}