<jupyter_start><jupyter_text>1. parsing dokumen<jupyter_code>## read each chats session folder path
""" 1. list folder chatx DONE
    2. list file dalam folder
    
    EXPECTED:
    current dir + /parsed doc/ + chatx/ + sesix.txt    
"""
parsed_doc_path = '../1_parsing/parsed_doc/'
chat_folders = [chat_folder for chat_folder in os.listdir(parsed_doc_path) if not chat_folder.startswith('.')]

paths = []
for chat_folder in chat_folders:
    pattern = parsed_doc_path + chat_folder
    paths.append(pattern)
    
sesi_paths = []
for path in paths:
    for roots, dirs, files in os.walk(path):
        if "checkpoints" not in roots:
            for file in files:
                sesi_paths.append(roots + '/' + file)
## make list of dataframe for each chats session
sessions = list()
for sesi_path in sesi_paths:
    sesi = pd.read_csv(sesi_path, sep='|', names=['sender', 'messages', 'sentiment'])
    sesi.drop('sender', axis=1, inplace=True) ## drop sender column (since its not necessary)
    sessions.append(sesi)
sessions[0].head()<jupyter_output><empty_output><jupyter_text>2. lexical analysis<jupyter_code>## lexical analysis - cleaning
def cleaning(message):
    """ This function will remove unnecessaries values in the messages i.e links, emails, and punctuations
    """
    link_reg = r'(https?:\/\/[^\s]+)|(www\.[^\s]+)|(meet\.google\.[^\s]+)|(bit\.ly[^\s]+)'
    email_reg = r'([a-zA-Z0-9\.\_\-]+@+[a-zA-Z0-9.]+)'
    punct_reg = r'[^a-zA-Z0-9\[\]]'
    numb_reg = r'\b[0-9]+\b\s*'
    
    message = re.sub(link_reg, '[LINK]', message) # -> link removed
    message = re.sub(email_reg, '', message) # -> email removed
    message = re.sub(punct_reg, ' ', message) # -> punctuation removed
    message = re.sub(numb_reg, '', message) # -> numbers removed
    
    return message
## lexical analysis - lower, except <MHS>, <DOSEN>, <LINK>
def lower(word_list):
    new_word_list = list()
    for word in word_list:
        if word != '[MHS]' and word != '[DOSEN]' and word != '[LINK]':
            word = word.lower()
        new_word_list.append(word)
    return new_word_list
## do lexical analysis
for sesi in sessions:
    for i, row in sesi.iterrows():
        message = row['messages']
        # do cleaning
        message = cleaning(message)
        # do strip
        message = message.strip()
        # do tokenization
        message = re.split('\s+', message)
        # do lower case
        message = lower(message)
        
        # replace current message value in df
        sesi.at[i, 'messages'] = message
sessions[0].head()<jupyter_output><empty_output><jupyter_text>3. normalization (slang word)<jupyter_code>## normalize function
def normalize(slang_words, tokens):
    """ This function will normalize the tokens,
        it will turn the slang words or typos to its normal values.
        NOTE: you could add the values into json files.
    """
    new_tokens = list()
    for token in tokens:
        new_value = slang_words.get(token, token) #2nd parameter for default value if word's key not found
        new_tokens.append(new_value)
    
    return new_tokens
## open slang_words json
f = open('slang_words.json', 'r')
slang_words = json.load(f)
f.close()

## do normalize
for sesi in sessions:
    for i, row in sesi.iterrows():
        tokens = row['messages']
        # do normalize
        new_tokens = normalize(slang_words, tokens)

        #replace current message value in df
        sesi.at[i, 'messages'] = new_tokens
sessions[0].head()<jupyter_output><empty_output><jupyter_text>4. filtering<jupyter_code>## filtering function
def filtering(stopword_list, tokens):
    """ Filtering: removing stopwords from tokens.
        In this project, we will use tala stopwords list.
    """
    new_tokens = list()
    for token in tokens:
        if token not in stopword_list:
            new_tokens.append(token)
    
    return new_tokens
## open stopword list txt
f = open('tala_stopwords.txt', 'r')
stopword_list = f.read()
f.close()

## do normalize
for sesi in sessions:
    for i, row in sesi.iterrows():
        tokens = row['messages']
        # do filtering
        new_tokens = filtering(stopword_list, tokens)

        #replace current message value in df
        sesi.at[i, 'messages'] = new_tokens
sessions[0].head()<jupyter_output><empty_output><jupyter_text>5. stemming<jupyter_code>## stemming function, except [DOSEN], [MHS], [LINK]
def stemming(stemmer, tokens):
    """ Stemming: returns words to its original form.
        Since non-alphanumeric will be discarded by using StemmerFactory(),
        This function will do stemming if the token values neither [dosen] nor [mhs].
    """
    new_tokens = list()
    for token in tokens:
        if token != '[MHS]' and token != '[<PASSWORD>]' and token != '[LINK]':
            token = stemmer.stem(token)
        new_tokens.append(token)
    
    return new_tokens
## create stemmer object
stemmer = StemmerFactory().create_stemmer()

## do normalize
for sesi in sessions:
    for i, row in sesi.iterrows():
        tokens = row['messages']
        # do filtering
        new_tokens = stemming(stemmer, tokens)

        # replace current message value in df
        sesi.at[i, 'messages'] = new_tokens
sessions[0].head()<jupyter_output><empty_output><jupyter_text>6. add session index for each chat, also make array of df into single df, using pandas concat<jupyter_code>## add index session column
for i, sesi in enumerate(sessions):
    index = [i for j in range(sesi.shape[0])]
    sesi['session_index'] = index

## concat all df into single df
sessions_final = pd.concat(sessions, ignore_index=True)<jupyter_output><empty_output><jupyter_text>7. drop empty preprocessing result's message<jupyter_code>empty_index = list()
for i, row in sessions_final.iterrows():
    if not row['messages']:
        empty_index.append(i)

print(empty_index)
sessions_final = sessions_final.drop(sessions_final.index[empty_index])<jupyter_output><empty_output><jupyter_text>8. drop empty word in result's message<jupyter_code>for i, row in sessions_final.iterrows():
    message = row['messages']
    message = filter(None, message)
    message = list(message)
    sessions_final.at[i, 'messages'] = message
sessions_final.iloc[1238]['messages']
## check again the empty index
empty_index = list()
for i, row in sessions_final.iterrows():
    if not row['messages']:
        empty_index.append(i)

if not empty_index:
    print('no empty message!')<jupyter_output><empty_output><jupyter_text>8. export to csv<jupyter_code>sessions_final.to_csv('preprocessing_results.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1250944251397492, 'fraction_numerical': 0.004834567155159389, 'mean_word_length': 3.4669365721997303, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '117928', 'license': 'MIT', 'n_tokens_mistral': 2185, 'n_tokens_neox': 2135, 'n_words': 588, 'path': '2_preprocessing/.ipynb_checkpoints/preprocessing-checkpoint.ipynb', 'repo_name': 'annisann/sentiment-analysis-for-conversational-texts', 'size': '6614'}