<jupyter_start><jupyter_text>read all and split into parts<jupyter_code>df = get_full_from_cache(INTERIM_DATA_ROOT,with_contents=True)
len(df)
df.head(10000).to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part1.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[10000:20000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part2.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[20000:30000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part3.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[30000:40000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part4.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[40000:50000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part5.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[50000:60000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part6.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[60000:70000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part7.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[70000:80000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part8.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[80000:90000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part9.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[90000:100000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part10.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[100000:110000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part11.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[110000:120000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part12.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[120000:130000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part13.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[130000:140000].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part14.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)
df.iloc[140000:].to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part15.csv",
    quoting=csv.QUOTE_ALL,
    line_terminator='\n',
    index=False)<jupyter_output><empty_output><jupyter_text>read, clean and write back each part<jupyter_code># part 1
df = pd.read_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part1.csv",
    lineterminator='\n')
df["contents"] =df["contents"].map(lambda text: clean_text_delicious(str(text)))
df.to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part1.csv",
    quoting=csv.QUOTE_ALL,
    index=False)
# part 2 to part 14
for i in range(2,15):   
    df = pd.read_csv(
        PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part{}.csv".format(i),
        lineterminator='\n')
    
    df["contents"] =df["contents"].map(lambda text: clean_text_delicious(str(text)))
    df.to_csv(
        PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part{}.csv".format(i),
        quoting=csv.QUOTE_ALL,
        header=False,
        index=False)
# part 15
df = pd.read_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part15.csv".format(i),
    lineterminator='\n')

df["contents"] =df["contents"].map(lambda text: clean_text_delicious(str(text)))
df.to_csv(
    PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part15.csv".format(i),
    quoting=csv.QUOTE_ALL,
    header=False,
    index=False)
# join all parts using cat
from subprocess import run,PIPE

parts = [PROCESSED_DATA_ROOT+"/delicious-t-140-with-contents-part-{}.csv".format(part) for part in range(1,16)]

output_file = PROCESSED_DATA_ROOT+"/delicious-t-140-with-clean-contents.csv"

command = ["/bin/cat"] + parts + [">",output_file]

# print(command)

run(command,stdout=PIPE)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1589840075258702, 'fraction_numerical': 0.06067732831608655, 'mean_word_length': 4.466580976863753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '465743', 'license': 'MIT', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1694, 'n_words': 163, 'path': 'social-tags/notebooks/helper-scripts/clean-delicious-text-and-save-csv.ipynb', 'repo_name': 'queirozfcom/auto-tagger', 'size': '4252'}