<jupyter_start><jupyter_text>raw_tf.ipynbcode for make raw tf csv file<jupyter_code>import pandas as pd<jupyter_output><empty_output><jupyter_text>1. convert messages into list<jupyter_code>chats_data = pd.read_csv('../2_preprocessing/preprocessing_results.csv')
chats_data.head(10)
messages_list = list()

for message in chats_data['messages']:
    
    message = message.replace('[', '').replace('\'', '').replace(']', '')
    message_list = message.split(', ')
    
    for i, each in enumerate(message_list):
        if each == 'DOSEN' or each == 'MHS' or each == 'LINK':
            each = f'[{each}]'
            message_list[i] = each
            
    messages_list.append(message_list)
print('total =', len(messages_list), 'chats')
print(messages_list[0:5])<jupyter_output>total = 1505 chats
[['[DOSEN]', 'mohon', 'maaf', 'ganggu', 'nama', '[MHS]', 'angkat', 'sedia', 'bimbing', 'skripsi', 'milik', 'topik', 'sentimen', 'analisis', 'produk', 'layan', 'indihome', 'metode', 'nearest', 'neighbor', 'information', 'topik', 'angkat', 'teliti', 'skripsi', 'terima', 'kasih'], ['sedia', 'topik', 'layak', 'posisi', 'map', 'kaji', 'pustaka'], ['terima kasih', 'mohon', 'maaf', 'mekanisme', 'bimbing', 'daring'], ['[MHS]', 'silah', 'kirim', 'dokumen', 'bimbing', 'minimal', 'minggu', 'pra', 'proposal', 'p0', 'p1', 'p2', 'seminar hasil', 'sidang', 'tolong', 'cantum', 'detail', 'content', 'email', 'lampir', 'draft', 'dokumen', 'dokumen', 'skripsi', 'dlm', 'doc', 'docx', 'power point', 'p0', 'p1', 'p2', 'seminar hasil', 'sidang', 'p0', 'p1', 'p2', 'seminar hasil', 'sidang', 'contoh', 'power point', 'link', 'grup', 'kelas', 'tinggal', 'ganti', 'isi', 'contoh', 'screenshoot', 'hasil', 'running', 'program', 'deskripsi', 'poin', 'revisi', 'deskripsi', 'poin', 'progres', 'selesai[...]<jupyter_text>2. make token set list from messages list<jupyter_code>token_set_list = list()
if messages_list:
    for message in messages_list:
        for word in message:
            if word not in token_set_list:
                token_set_list.append(word)
print('total =', len(token_set_list), 'unique words')
print(token_set_list[:5])<jupyter_output>total = 1740 unique words
['[DOSEN]', 'mohon', 'maaf', 'ganggu', 'nama']<jupyter_text>3. count raw tf each word for each chat<jupyter_code>count_all_message = list()
for message in messages_list:
    count_per_message = list()
    for token in token_set_list:
        tf_token = message.count(token)
        count_per_message.append(tf_token)
    count_all_message.append(count_per_message)
print('total message counted =', len(count_all_message))
print('total word counted from all message =', len(count_all_message[-1]))<jupyter_output>total message counted = 1505
total word counted from all message = 1740<jupyter_text>4. make dataframe for raw tf<jupyter_code>result_df = pd.DataFrame({'tokens':token_set_list})
for i, count_per_message in enumerate(count_all_message):
    column_title = f'chat_{i}'
    result_df[column_title] = count_per_message<jupyter_output><empty_output><jupyter_text>5. export df to csv<jupyter_code>result_df.to_csv('raw_tf.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18410174880763117, 'fraction_numerical': 0.011764705882352941, 'mean_word_length': 5.809523809523809, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '382880', 'license': 'MIT', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1166, 'n_words': 258, 'path': '3_tf_idf/raw_tf.ipynb', 'repo_name': 'annisann/sentiment-analysis-for-conversational-texts', 'size': '3145'}