<jupyter_start><jupyter_text>Contains:- Preprocess- Load model- Predict<jupyter_code>!pip install emoji
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import json
import requests
import string
import re
import functools
import operator

from joblib import dump, load

import emoji

from nltk import word_tokenize, sent_tokenize
import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('punkt')<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!<jupyter_text>Build json file for kamus alay, resources:- https://github.com/ramaprakoso/analisis-sentimen- https://github.com/nasalsabila/kamus-alay<jupyter_code># build single file for kamus alay
# CONSTRUCT KAMUS ALAY
text_path1 = 'https://raw.githubusercontent.com/ramaprakoso/analisis-sentimen/master/kamus/kbba.txt'
text_path2 = 'https://raw.githubusercontent.com/nasalsabila/kamus-alay/master/colloquial-indonesian-lexicon.csv'
kamus_alay1 = pd.read_csv(text_path1, delimiter="\t", header=None, names=['slang', 'formal'])
kamus_alay2 = pd.read_csv(text_path2)
kamus_alay = pd.concat([kamus_alay1, kamus_alay2[['slang', 'formal']]]).reset_index(drop=True)

dict_alay = dict()
for index, row in kamus_alay.iterrows():
    dict_alay[row['slang']] = row['formal']

# with open('kamus_alay.json', 'w') as fp:
#     json.dump(dict_alay, fp)<jupyter_output><empty_output><jupyter_text>Build list of stop words, resources:- https://github.com/ramaprakoso/analisis-sentimen- https://github.com/yasirutomo/python-sentianalysis-id- https://github.com/onlyphantom/elangdev- Sastrawi- Fpmipa- Ali Akbar- Pebbie- NLTK- Custom stop words<jupyter_code># CONSTRUCT STOPWORDS
rama_stopword = "https://raw.githubusercontent.com/ramaprakoso/analisis-sentimen/master/kamus/stopword.txt"
yutomo_stopword = "https://raw.githubusercontent.com/yasirutomo/python-sentianalysis-id/master/data/feature_list/stopwordsID.txt"
fpmipa_stopword = "https://raw.githubusercontent.com/onlyphantom/elangdev/master/elang/word2vec/utils/stopwords-list/fpmipa-stopwords.txt"
sastrawi_stopword = "https://raw.githubusercontent.com/onlyphantom/elangdev/master/elang/word2vec/utils/stopwords-list/sastrawi-stopwords.txt"
aliakbar_stopword = "https://raw.githubusercontent.com/onlyphantom/elangdev/master/elang/word2vec/utils/stopwords-list/aliakbars-bilp.txt"
pebahasa_stopword = "https://raw.githubusercontent.com/onlyphantom/elangdev/master/elang/word2vec/utils/stopwords-list/pebbie-pebahasa.txt"
elang_stopword = "https://raw.githubusercontent.com/onlyphantom/elangdev/master/elang/word2vec/utils/stopwords-id.txt"
nltk_stopword = stopwords.words('indonesian')

path_stopwords = [rama_stopword, yutomo_stopword, fpmipa_stopword, sastrawi_stopword, 
                  aliakbar_stopword, pebahasa_stopword, elang_stopword]

# CUSTOM STOPWORDS
other = '''
admin mimin min minkes kalo nya username
'''

# gabungkan stopwords
stopwords_l = nltk_stopword
for path in path_stopwords:
    response = requests.get(path)
    stopwords_l += response.text.split('\n')

st_words = set(stopwords_l)
other_stopword = set(other.split())

stop_words = sorted(list(st_words | other_stopword))
stop_words = stop_words[1:]

with open('stop_words.txt', 'w') as file:
    for i in range(len(stop_words)):
        if i == len(stop_words)-1:
            file.write(stop_words[i])
        else:
            file.write(stop_words[i] + '\n')<jupyter_output><empty_output><jupyter_text>Preprocess PipelineYou need to import these files:- emoji_to_text.csv- kamus_alay.json- stop_words.txt<jupyter_code># CLEANSING
def cleansing(data):
    # lowercasing
    data = data.lower()
    # remove punctuation
    punct = string.punctuation
    translator = str.maketrans(punct, ' '*len(punct))
    data = data.translate(translator)
    # remove newline
    data = data.replace('\n', ' ')
    # remove digit
    pattern = r'[0-9]'
    data = re.sub(pattern, '', data)
    # remove extra space
    data = ' '.join(data.split())
    return data

# CONVERT EMOJIS
df_emoji = pd.read_csv('emoji_to_text.csv')
UNICODE_EMO = {row['emoji']:row['makna'] for idx,row in df_emoji.iterrows()}
def convert_emojis(text):
    # split emojis
    em_split_emoji = emoji.get_emoji_regexp().split(text)
    em_split_whitespace = [substr.split() for substr in em_split_emoji]
    em_split = functools.reduce(operator.concat, em_split_whitespace)
    text = ' '.join(em_split)
    # convert emojis
    for emot in UNICODE_EMO:
        text = re.sub(r'('+emot+')', "_".join(UNICODE_EMO[emot].replace(",","").replace(":","").split()), text)
    return text.lower()

# NORMALIZE COLLOQUIAL/ALAY
kamus_alay = json.load(open('kamus_alay.json', 'r'))
def normalize_text(data):
  word_tokens = word_tokenize(data)
  result = [kamus_alay.get(w,w) for w in word_tokens]
  return ' '.join(result)

# REMOVE STOPWORDS
stop_words = [sw.strip() for sw in open('stop_words.txt', 'r').readlines()]
def remove_stopword(text, stop_words=stop_words):
    word_tokens = word_tokenize(text)
    filtered_sentence = [w for w in word_tokens if not w in stop_words]
    return ' '.join(filtered_sentence)
# PREPROCESS PIPELINE
def preprocess(text):
    text = cleansing(text)
    text = convert_emojis(text)
    text = normalize_text(text)
    text = remove_stopword(text)
    return text
sample_text1 = "Ayooo.. Tetep ProKes ketat..!!! Janhan lengah..!!! Semangat...!!!"
sample_text2 = "WKWK SIYAP MAS GREGAH 😂🔥👏"

text1 = preprocess(sample_text1)
text2 = preprocess(sample_text2)

print(preprocess(text1))
print(preprocess(text2))<jupyter_output>ayo prokes ketat janhan lengah semangat
tertawa siyap mas gregah wajah tertawa menangis api bertepuk tangan<jupyter_text>Feature Extraction Pipeline  TF-IDF PipelineYou need to import this joblib file:- tfidf-vectorizer.joblib<jupyter_code># load tf-idf vectorizer model
vectorizer = load('tfidf-vectorizer.joblib')
# feature extraction/transform text
feature1 = vectorizer.transform([text1])
feature2 = vectorizer.transform([text2])

print(feature1.shape)
print(feature1.toarray())
print(feature2.toarray())
feature1.toarray().max()<jupyter_output><empty_output><jupyter_text>Word2Vec Pipeline<jupyter_code># dowload pre-trained word2vec fasttext indonesia
! wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.bin.gz
# unzip
! gunzip cc.id.300.bin.gz
from gensim.models import KeyedVectors
from gensim.models.wrappers import FastText

# load pre-trained word2vec fasttext
word2vec = FastText.load_fasttext_format('cc.id.300.bin')
# VECTORIZE
VOCABULARY = word2vec.wv.vocab
def vectorize(tokens):
    word_vec = []
    for w in tokens:
        if w in VOCABULARY:
            # get word vector from pre-trained word2vec fasttext
            word_vec.append(word2vec[w])
    return word_vec

def avg_vectorize(tokens):
    sum_vec = np.zeros(300)
    word_count = 0
    for w in tokens:
        if w in VOCABULARY:
            # word vector from pre-trained word2vec fasttext and add vector
            sum_vec += word2vec[w]
            word_count += 1
    return sum_vec if word_count==0 else sum_vec/word_count

# PADDING
MAX_LEN = 50
def add_padding(word_vec):
    if len(word_vec) < MAX_LEN:
        pad_count = MAX_LEN - len(word_vec)
        return word_vec + [np.array([0]*300)]*pad_count
    else:
        return word_vec[:MAX_LEN]

# TOKENIZE
def tokenize(text):
    return word_tokenize(text)
# feature extraction
def extract_feature(text, ndim=3):
    if ndim == 3:  # for deep learning model
        features = vectorize(text)
        features = add_padding(features)
    elif ndim == 2:  # for sklearn model
        features = avg_vectorize(text)
    return np.array(features)
tokenize(text1)
# tokenize
text1 = tokenize(text1)

feat1_dim3 = extract_feature(text1, ndim=3)
feat1_dim2 = extract_feature(text1, ndim=2)

feat1_dim3.shape, feat1_dim2.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1196071117742136, 'fraction_numerical': 0.010070869078701977, 'mean_word_length': 4.749821300929235, 'pattern_counts': {'":': 1, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 15, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '457044', 'license': 'Apache-2.0', 'n_tokens_mistral': 3007, 'n_tokens_neox': 2942, 'n_words': 557, 'path': 'notebooks/utils/Research_DKK_Semarang_Create_Function_for_Deployment.ipynb', 'repo_name': 'datascienceid/project-sentiment-dkk-semarang', 'size': '8043'}