<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import pandas as pd<jupyter_output><empty_output><jupyter_text>Read Config File<jupyter_code>import configparser
config = configparser.ConfigParser()
config.read('config.ini')
input_file = config['DEFAULT']['Input-File']
output_file = config['DEFAULT']['Output-File']
column_name = config['DEFAULT']['Column-Name']
split_character = config['DEFAULT']['Split-Character']
rename = config['DEFAULT']['Rename']<jupyter_output><empty_output><jupyter_text>Read CSV<jupyter_code>df = pd.read_csv(input_file, header=0, encoding="UTF-8")<jupyter_output><empty_output><jupyter_text>Get Value from Column<jupyter_code>column = df[column_name].tolist()<jupyter_output><empty_output><jupyter_text>Get New Column List From Column Value<jupyter_code>new_column_list = []
for i in column:
    if isinstance(i, str):
        i_split = i.split(split_character)
        for j in i_split:
            j = j.lstrip()
            if j.lower() not in new_column_list:
                new_column_list.append(j.lower())
new_column_list.sort()<jupyter_output><empty_output><jupyter_text>Add New Header<jupyter_code>for i in new_column_list:
    df[i] = None<jupyter_output><empty_output><jupyter_text>Append New Value<jupyter_code># iterate rows to append new value
for index,row in df.iterrows():
    value_list = []
    value = df.loc[index,column_name]
    
    for i in new_column_list:
        if isinstance(value, str):
            if i.lower() in value.lower():
                value_list.append("x")
            else:
                value_list.append(None)  
        else:
            value_list.append(None)
    df.loc[[index],new_column_list] = value_list<jupyter_output><empty_output><jupyter_text>Rename Column<jupyter_code>if rename == "1":
    for i in new_column_list:
        df.rename(columns={i:column_name + ":" + i}, inplace=True)<jupyter_output><empty_output><jupyter_text>Delete Original Column<jupyter_code>del df[column_name]<jupyter_output><empty_output><jupyter_text>Write into CSV<jupyter_code># write into csv
df.to_csv(output_file, sep=',',index = False, encoding='UTF-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12748267898383372, 'fraction_numerical': 0.0018475750577367205, 'mean_word_length': 5.170940170940171, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '657584', 'license': 'Unlicense', 'n_tokens_mistral': 757, 'n_tokens_neox': 748, 'n_words': 119, 'path': 'f0018/f0018-a.ipynb', 'repo_name': 'soda-lab/Features', 'size': '2165'}