<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>Mengambil dan Mengekstrak Datasethttps://github.com/kmkurn/id-pos-tagging/blob/master/data<jupyter_code>!wget  https://github.com/kmkurn/id-pos-tagging/blob/master/data/dataset.tar.gz?raw=true
# open file
import tarfile

file = tarfile.open('dataset.tar.gz?raw=true')
  
# extracting file
file.extractall('dataset')
  
file.close()
# # Importing libraries
# import nltk
# import numpy as np
# import pandas as pd
# import random
# from sklearn.model_selection import train_test_split
# import pprint, time
 
# #download the treebank corpus from nltk
# nltk.download('treebank')
 
# #download the universal tagset from nltk
# nltk.download('universal_tagset')
 
# # reading the Treebank tagged sentences
# nltk_data = list(nltk.corpus.treebank.tagged_sents(tagset='universal'))
 
# #print the first two sentences along with tags
# print(nltk_data[:2])<jupyter_output><empty_output><jupyter_text>Mengambil Data Training<jupyter_code>import csv
with open('/content/dataset/train.01.tsv', newline='') as f:
    reader = csv.reader(f,delimiter='\t',quoting=csv.QUOTE_NONE)
    train_set = list(reader)

print(train_set[0:40])
print(len(train_set))<jupyter_output>[['Pemerintah', 'NN'], ['bahkan', 'RB'], ['telah', 'MD'], ['mencanangkan', 'VB'], ['dana', 'NN'], ['untuk', 'SC'], ['memicu', 'VB'], ['sektor', 'NN'], ['usaha', 'NN'], ['kecil', 'JJ'], ['dan', 'CC'], ['menengah', 'JJ'], ['UKM', 'NN'], ['tumbuh', 'VB'], ['lebih', 'RB'], ['baik', 'JJ'], [',', 'Z'], ['karena', 'SC'], ['sektor', 'NN'], ['ini', 'PR'], ['cukup', 'RB'], ['kuat', 'JJ'], ['dalam', 'IN'], ['krisis', 'JJ'], ['keuangan', 'NN'], ['pada', 'IN'], ['tahun', 'NN'], ['1997', 'CD'], ['lalu', 'CC'], [',', 'Z'], ['kata', 'VB'], ['-nya', 'PRP'], ['.', 'Z'], [], ['Gubernur', 'NNP'], ['Bank', 'NNP'], ['Indonesia', 'NNP'], ['Burhanudin', 'NNP'], ['Abdullah', 'NNP'], ['optimistis', 'JJ']]
191567<jupyter_text>Mengambil Data Testing<jupyter_code>with open('/content/dataset/test.01.tsv', newline='') as f:
    reader = csv.reader(f,delimiter='\t')
    test_set = list(reader)
print(test_set[0:30])
print(len(test_set))<jupyter_output>[['Pemerintah', 'NNP'], ['AS', 'NNP'], [',', 'Z'], ['pimpinan', 'NN'], ['Presiden', 'NNP'], ['George', 'NNP'], ['W.', 'NNP'], ['Bush', 'NNP'], [',', 'Z'], ['dalam', 'IN'], ['beberapa', 'CD'], ['pekan', 'NN'], ['belakangan', 'NN'], ['telah', 'MD'], ['meningkatkan', 'VB'], ['pendirian', 'NN'], ['-nya', 'PRP'], ['terhadap', 'IN'], ['Iran', 'NNP'], [',', 'Z'], ['yang', 'SC'], ['telah', 'MD'], ['dituduh', 'VB'], ['Barat', 'NNP'], ['secara', 'IN'], ['diam-diam', 'RB'], ['sedang', 'MD'], ['berusaha', 'VB'], ['membuat', 'VB'], ['bom atom', 'NN']]
53481<jupyter_text>Preprocessing1.   Semua huruf jadi huruf kecil cth: LAsd -> lasd2.   Semua angka menjadi 0 cth: dsaq123 -> dsaq0<jupyter_code>def transform(w, lower=True, replace_digits=True):
    
    if lower:
        w = w.lower()
    if replace_digits:
        w = re.sub(r'\d+', '0', w)
    return w<jupyter_output><empty_output><jupyter_text>Membersihkan data yang kosong dan menyesuaikan format<jupyter_code># create list of train and test tagged words
# train_tagged_words = [ tup for sent in train_set for tup in sent ]
# test_tagged_words = [ tup for sent in test_set for tup in sent ]
# print(len(train_tagged_words))
# print(len(test_tagged_words))
import regex as re

def Remove(tuples):
    tuples = [t for t in tuples if t]
    return tuples

train_set = Remove(train_set)
test_set = Remove(test_set)

train_tagged_words = []
for alist in train_set:
    alist[0] = transform(alist[0])
    
    train_tagged_words.append(tuple(alist))

test_tagged_words = []
for alist in test_set:
    alist[0] = transform(alist[0])
    test_tagged_words.append(tuple(alist))
print(train_tagged_words[:5])
print(test_tagged_words[:5])<jupyter_output>[('pemerintah', 'NN'), ('bahkan', 'RB'), ('telah', 'MD'), ('mencanangkan', 'VB'), ('dana', 'NN')]
[('pemerintah', 'NNP'), ('as', 'NNP'), (',', 'Z'), ('pimpinan', 'NN'), ('presiden', 'NNP')]<jupyter_text>Mengecek jenis kata dan vocabulary<jupyter_code>tags = {tag for word,tag in train_tagged_words}
print(len(tags))
print(tags)
 
# check total words in vocabulary
vocab = {word for word,tag in train_tagged_words}
print(len(vocab))<jupyter_output>23
{'CD', 'SYM', 'DT', 'RP', 'Z', 'PRP', 'CC', 'WH', 'RB', 'OD', 'SC', 'NN', 'NNP', 'IN', 'VB', 'PR', 'NND', 'NEG', 'X', 'JJ', 'UH', 'MD', 'FW'}
11680<jupyter_text>Fungsi untuk mengecek Peluang<jupyter_code># compute Emission Probability
def word_given_tag(word, tag, train_bag = train_tagged_words):
    tag_list = [pair for pair in train_bag if pair[1]==tag]
    count_tag = len(tag_list)#total number of times the passed tag occurred in train_bag
    w_given_tag_list = [pair[0] for pair in tag_list if pair[0]==word]
#now calculate the total number of times the passed word occurred as the passed tag.
    count_w_given_tag = len(w_given_tag_list)
 
     
    return (count_w_given_tag, count_tag)

# compute  Transition Probability
def t2_given_t1(t2, t1, train_bag = train_tagged_words):
    tags = [pair[1] for pair in train_bag]
    count_t1 = len([t for t in tags if t==t1])
    count_t2_t1 = 0
    for index in range(len(tags)-1):
        if tags[index]==t1 and tags[index+1] == t2:
            count_t2_t1 += 1
    return (count_t2_t1, count_t1)<jupyter_output><empty_output><jupyter_text>Membuat Model Peluang HMM<jupyter_code>import numpy as np
import pandas as pd

tags_matrix = np.zeros((len(tags), len(tags)), dtype='float32')
for i, t1 in enumerate(list(tags)):
    for j, t2 in enumerate(list(tags)): 
        tags_matrix[i, j] = t2_given_t1(t2, t1)[0]/t2_given_t1(t2, t1)[1]

tags_df = pd.DataFrame(tags_matrix, columns = list(tags), index=list(tags))
display(tags_df)
import seaborn as sns
import matplotlib.pyplot as plt

plt.figure(figsize = (25,16))

ax = sns.heatmap(tags_df, linewidths=.1,annot=True)
ax.xaxis.tick_top()
ax.tick_params(length=0)<jupyter_output><empty_output><jupyter_text>Membuat Fungsi Viterbi<jupyter_code>def Viterbi(words, train_bag = train_tagged_words):
    state = []
    T = list(set([pair[1] for pair in train_bag]))
     
    for key, word in enumerate(words):
        #initialise list of probability column for a given observation
        p = [] 
        for tag in T:
            if key == 0:
                transition_p = tags_df.loc['Z', tag]
            else:
                transition_p = tags_df.loc[state[-1], tag]
                 
            # compute emission and state probabilities
            emission_p = word_given_tag(words[key], tag)[0]/word_given_tag(words[key], tag)[1]
            state_probability = emission_p * transition_p    
            p.append(state_probability)
             
        pmax = max(p)
        # getting state for which probability is maximum
        state_max = T[p.index(pmax)] 
        state.append(state_max)
    return list(zip(words, state))<jupyter_output><empty_output><jupyter_text>Testing untuk 10 kata pertama pada data test<jupyter_code>import time
# list of 10 sents on which we test the model
test_run = test_set[0:10]
 
# list of tagged words
test_run_base = [tuple(sent) for sent in test_run]
 
# list of untagged words
test_tagged_words = [sent[0] for sent in test_run]

#Here We will only test 10 sentences to check the accuracy
#as testing the whole training set takes huge amount of time
start = time.time()
tagged_seq = Viterbi(test_tagged_words)
end = time.time()
difference = end-start
 
print("Time taken in seconds For The First 10th Word: ", difference)
 
# accuracy
check = [i for i, j in zip(tagged_seq, test_run_base) if i == j] 
 
accuracy = len(check)/len(tagged_seq)
print('Viterbi Algorithm Accuracy For The First 10th Word: ',accuracy*100)<jupyter_output>Time taken in seconds For The First 10th Word:  7.930094480514526
Viterbi Algorithm Accuracy For The First 10th Word:  90.0<jupyter_text>Testing untuk semua data test (Dikarenakan terbatasnya google colab, maka testing dilakukan secara perlahan)<jupyter_code>#Code to test all the test sentences
#(takes alot of time to run s0 we wont run it here)
# tagging the test sentences()
from pathlib import Path
import csv

test_tagged_words = [tuple(sent) for sent in test_set]
test_untagged_words = [sent[0] for sent in test_set]
#test_untagged_words

interval = 2000
filename = '/content/drive/MyDrive/output.csv'
if Path(filename).is_file():
    with open(filename, newline='') as f:
        reader = csv.reader(f)
        listtagged_seq = list(reader)
    prevtagged_seq = []
    for i in listtagged_seq:
        prevtagged_seq.append(tuple(i))
    startidx = len(prevtagged_seq)
else:
    startidx = 0
    prevtagged_seq = []

endinterval = startidx + interval #(titik pada kalimat terakhir)
if endinterval > len(test_untagged_words):
    endinterval = len(test_untagged_words)-1


for i in range(endinterval,len(test_untagged_words)):
    if test_untagged_words[i] == '.':
        break
endinterval = i+1

if len(prevtagged_seq) < len(test_tagged_words):


    start = time.time()
    current = start

    tagged_seq = Viterbi(test_untagged_words[startidx:endinterval])
    end = time.time()
    difference = end-start
    wordcount = endinterval - startidx
    print("Finish at index: ", endinterval)
    print("Number of Word Prcessed: ", wordcount)
    print("Time taken in seconds: ", difference)


    # accuracy
    check = [i for i, j in zip(tagged_seq, test_tagged_words[startidx:endinterval]) if i == j] 
    
    accuracy = len(check)/len(tagged_seq)
    print('Viterbi Algorithm Accuracy: ',accuracy*100)



    prevtagged_seq.extend(tagged_seq)
    listouput = []
    for tuples in prevtagged_seq:
        listouput.append(list(tuples))
    with open(filename, "w") as f:
        writer = csv.writer(f)
        writer.writerows(listouput)
if len(prevtagged_seq) >= len(test_tagged_words):
    print("Part Of Speech Tagging has Been Finished")<jupyter_output>Part Of Speech Tagging has Been Finished<jupyter_text>Hasil Akurasi dari prediksi POS Tagging pada semua data test<jupyter_code>with open(filename, newline='') as f:
    reader = csv.reader(f)
    listtagged_seq = list(reader)
    prevtagged_seq = []
for i in listtagged_seq:
    prevtagged_seq.append(tuple(i))

check = [i for i, j in zip(prevtagged_seq, test_tagged_words[0:len(prevtagged_seq)]) if i == j] 
 
accuracy = len(check)/len(test_tagged_words[0:len(prevtagged_seq)])
print('Viterbi Algorithm Accuracy: ',accuracy*100)<jupyter_output>Viterbi Algorithm Accuracy:  93.69985429820301<jupyter_text>NLTK untuk Word Tokenize<jupyter_code>import nltk
nltk.download('punkt')<jupyter_output><empty_output><jupyter_text>Testing untuk kalimat Inputan<jupyter_code>print('Masukkan Kalimat yang akan di Post Tagging: ')
sentence = input()
sentence = nltk.word_tokenize(sentence)
sentence = list(map(transform, sentence))
taggedsentence = Viterbi(sentence)
print(taggedsentence)<jupyter_output>Masukkan Kalimat yang akan di Post Tagging: 
Amerika tidak jadi membeli hand sanitizer seharga 5000000$ dari negara lain. Ledakan yang terjadi di prancis dikarenakan kelalaian petugas.
[('amerika', 'NNP'), ('tidak', 'NEG'), ('jadi', 'VB'), ('membeli', 'VB'), ('hand', 'CD'), ('sanitizer', 'CD'), ('seharga', 'NN'), ('0', 'CD'), ('$', 'SYM'), ('dari', 'IN'), ('negara', 'NN'), ('lain', 'JJ'), ('.', 'Z'), ('ledakan', 'NN'), ('yang', 'SC'), ('terjadi', 'VB'), ('di', 'IN'), ('prancis', 'NNP'), ('dikarenakan', 'VB'), ('kelalaian', 'NN'), ('petugas', 'NN'), ('.', 'Z')]
{'fraction_non_alphanumeric': 0.16415338216286082, 'fraction_numerical': 0.015165876777251185, 'mean_word_length': 4.9123790117167605, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 56, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '513573', 'license': 'MIT', 'n_tokens_mistral': 4497, 'n_tokens_neox': 4222, 'n_words': 1050, 'path': 'PosTagging.ipynb', 'repo_name': 'rizkiar00/HMMPosTaggingIndonesia', 'size': '11605'}