<jupyter_start><jupyter_text>REPLACE SLANG WORDS<jupyter_code>import json

# Using Salsabila colloquial dictionary 
with open("kamus/nasalsabila_kamus-alay/_json_colloquial-indonesian-lexicon.txt") as f:
    data = f.read()
# Reconstructing the data as 'dict'
lookp_dict = json.loads(data)
import os

# Be sure to change the path to absolute path of your directory if you re-run, or restart the kernel instead.
os.chdir("output")
base = "prastyo-sentiment_posneg-clean.txt"

# Open input file and read it line by line
input_stream = open(base, 'r')
input_stream_lines = input_stream.readlines()
input_stream.close()
# Separate text and label from the input
text = []
for line in input_stream_lines:
    text.append(line.split("\t")[0])
    
label = []
for line in input_stream_lines:
    label.append(line.split("\t")[1])
# Creating output file
output = os.path.splitext(base)[0]+'-slang.txt'

with open(output, 'w') as f:
    # Replacing each found word in input_file with its replacement
    for line in text:
        res = " ".join(lookp_dict.get(ele, ele) for ele in line.split())
        # Writing each replaced line to output file
        f.write(str(res)+'\n')<jupyter_output><empty_output><jupyter_text>REMOVE STOPWORDS<jupyter_code>base = output

# Open input file and read it 'word by word in each line'
input_stream = open(base, 'r')
lines = input_stream.read().splitlines()
input_stream.close()

# Using Wahid ID-Stopwords
# Be sure to change the path to absolute path of your directory if you re-run, or restart the kernel instead.
with open('../kamus/masdevid_id-stopwords/id.stopwords.02.01.2016.txt') as f:
    custom_stop = f.read().splitlines()
from nltk.corpus import stopwords

# utilizing nltk stopwords to use custom stopwords
stopwords = stopwords.words('indonesian')
stopwords.clear()
stopwords.extend(custom_stop)
print(stopwords[:3])
# Creating output file
output = os.path.splitext(base)[0]+'-stop.txt'

count=0
with open(output, 'w') as f:
    # Remove stopword found in line
    for line in lines:
        s = ""
        words = line.split()
        for w in words:
            if not w in stopwords:
                s+=str(w)+" "
        res = s

        # Writing each corrected line to output file
        # and returning its label
        f.write(str(res+"\t"+label[count]))
        count+=1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11384876805437553, 'fraction_numerical': 0.006372132540356839, 'mean_word_length': 3.7672064777327936, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '354828', 'license': 'MIT', 'n_tokens_mistral': 795, 'n_tokens_neox': 776, 'n_words': 231, 'path': 'ipynb-en/[2] slangword-stopword-removal.ipynb', 'repo_name': 'onpilot/sentiment-bahasa', 'size': '2354'}