<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
!pwd
cd /content/drive/My Drive/SWU-Master-Thesis-63/Ploy/topicmodeling_thai
import requests
from bs4 import BeautifulSoup
from time import  time, sleep
from random import randint

# from googletrans import Translator

from tqdm import tqdm
from tqdm import tqdm_notebook
import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt

from nltk import sent_tokenize, word_tokenize
from nltk.sentiment.vader import  SentimentIntensityAnalyzer
from nltk.sentiment.util import mark_negation
dataset = pd.read_csv("dataset/dataset_question_th.csv",encoding='utf8')
print(dataset.shape)
dataset.head()
print(dataset.shape)
print("===============================================")
print(dataset.info())
%reload_ext autoreload
%autoreload 2
%matplotlib inline
try:
    %tensorflow_version 2.x
except:
    pass
#dev version
# !pip install https://github.com/PyThaiNLP/pythainlp/archive/dev.zip

# release version 
! pip install pythainlp

!pip install epitran
!pip install sklearn_crfsuite
!pip install tensorflow deepcut
!pip install attacut
import warnings
warnings.filterwarnings('ignore')
import pythainlp

pythainlp.__version__
dataset_q = dataset.iloc[0:50,0]
import re, string, unicodedata
from pythainlp import sent_tokenize, word_tokenize
from pythainlp.corpus import thai_stopwords
from pythainlp.util import normalize
from pythainlp.spell import correct


stopwords = list(thai_stopwords())

def preprocess_review(reviews): 
    cleaned_words = []
    for review in reviews:
        review = re.sub("[^\u0E00-\u0E7Fa-zA-Z0-9' ]|^'|'$|''", "", str(review))
        words = word_tokenize(review, engine="deepcut")
        tokenized_words = [word for word in words if word not in stopwords]
        # correct_word = correct(str(tokenized_words))
        # lemmatized_word = lemmatize_word(tokenized_words)
        cleaned_words.append(tokenized_words)
    return cleaned_words


# data_np = np.array(str(data_stopw))
data_stopw = preprocess_review(dataset_q)
data_stopw

import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk.stem.porter import *
import numpy as np
np.random.seed(2018)
from multiprocessing import Pool
import nltk
nltk.download('wordnet')

# def lemmatize_stemming(text):
#     return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))
#     def preprocess(text):
#       result = []
#     for token in gensim.utils.simple_preprocess(text):
#         if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:
#             result.append(lemmatize_stemming(token))
#     return result
dataset = dataset[['question']]
dataset['index'] = dataset.index
dataset = dataset

from sklearn import preprocessing

doc_sample = dataset[dataset['index'] == 4310].values[0][0]
print('original document: ')
words = []
for word in doc_sample.split(' '):
      words.append(word)
print(words)
print('\n\n tokenized and lemmatized document: ')
print(simple_preprocess(doc_sample))
!pip install pathos

import pandas as pd
import numpy as np
import time
from pymongo import MongoClient
from bson.objectid import ObjectId

import datetime
import calendar
import scipy.stats
from datetime import timedelta, date
import pytz
from pytz import timezone
import multiprocessing
from multiprocessing import cpu_count #For Parallel


cores = cpu_count() #Number of CPU cores on your system
partitions = cores #Define as many partitions as you want to run parallel

def lemmatize_stemming(text):
    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))

def parallelize(data, func):
    data_split = np.array_split(data, partitions)
    pool = multiprocessing.Pool(cores)     #สร้าง multiprocessor จำนวนเท่ากับ core cpu ของ server
    data = pd.concat(pool.map(func, data_split))
    pool.close()
    pool.join()
    return data

#ตัดคำ
def preprocess(documents):
    list_words= ["สวัสดี","ขอบคุณ","อยู่แล้ว" , "สัปดาห์" ,"หนูก้อรีบ" ,"ชอบเป้น" ,"นิดนึง" ,"ตุลาเลย" , "เหมือน" , "สอบถาม", "อะครับ","คะอยาก","เสี่ยง","ปริมาณ","ต่อเนื่อง","เปื้อน","กางเกง",
                 "สุดท้าย","เรื่อง","รึป่าว","มากน้อย","ชั่วโมง","ประมาณ","บริเวณ","เชี่ยวชาญ","พยาบาล","ประมาน","นิดเดียว","สม่ำเสมอ","กคประจำ","วี่แวว","เที่ยง","คะแล้ว","ลักษณะ",
                 "อาทิตย์","จันทร์","อังคาร","พุธ","พฤหัส","ศุกร์","เสาร์","คะดิฉัน","สองสาม","รุ้ว่า","ปากช่อง","รึเปล่า","สังเกตุ","ทั้งๆที่","เป็นอยู่","มิถุนา","ได้ยิน","แน่นอน","เครื่อง","ตัดสินใจ","อวัยวะ","สมมุติ","เตรียม","ล่วงหน้า","เรียบร้อย","คับคือ",
                 "เมษายน","ธันวาคม","พฤษภาคม","พฤศจิกายน","มกราคม","กุมภาพันธ์","มีนาคม","มิถุนายน","สิงหาคม","กรกฎาคม","ตุลาคม","กันยายน",
                 "จังหวัด", "เลื่อน","ปรึกษา", "เกี่ยว","ต้องการ","รู้สึก","สมบูรณ์","มั่นใจ", "เนื่อง", "สำหรับ", "คะพอดี","รุ้สึก","หาสจาก","ร่างกาย","ห้องน้ำ"]
    result = []
    for token in word_tokenize(documents,engine='deepcut'): 
        token = token.strip()
        if token not in stopwords and len(token) >5 and token not in list_words:
            token = correct(token)
            result.append(lemmatize_stemming(token))
    return result


def func(documents):
    res = documents['headline_text'].map(preprocess)
    return res

def cleanText(documents):
    documents['headline_text'] = documents['headline_text'].str.replace('|',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('\\',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('\/',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('.',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('_','')
    documents['headline_text'] = documents['headline_text'].str.replace('\d+', '')
    documents['headline_text'] = documents['headline_text'].str.replace('-',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('+',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('+',' ')
    pattern = '|'.join(['&', '%', ';', '='])
    documents['headline_text'] = documents['headline_text'].str.replace(pattern, ' ')
    # documents = documents.loc[~documents['headline_text'].str.contains("img")]
    documents.headline_text.replace({r"[^ก-๙]":''}, regex=True, inplace=True)
    documents['headline_text'] = documents['headline_text'].str.replace(':',' ')
    documents['headline_text'] = documents['headline_text'].str.replace('#',' ')
    documents['headline_text'] = documents['headline_text'].str.strip()
    documents = documents.loc[documents['headline_text']!= ""]
    documents = documents.dropna().reset_index(drop=True)
    return documents

# documents = dataset.copy()
# documents = documents.rename(columns={'question':'headline_text'})

# # concatenate all string of same user to be headline_text
# documents = documents.reset_index()
# print(documents.shape[0])
# documents.head()
dataset = dataset.iloc[:,:]
documents = dataset.copy()
documents = documents.rename(columns={'question':'headline_text'})

# concatenate all string of same user to be headline_text
# documents = documents.agg(lambda col: ' '.join(col)).reset_index()
documents = documents.groupby(['headline_text'], as_index = False).agg({'headline_text': lambda x: ' '.join(x.astype(str))})
print(documents.shape[0])
documents.head()



documents = parallelize(documents,cleanText)
documents.head()

import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk.stem.porter import *
import numpy as np
np.random.seed(2018)
import nltk
from pythainlp.tokenize import word_tokenize


allLen = documents.shape[0]
progress = 0
# Print iterations progress
def printProgressBar (iteration, total,mode = 0,current_percent = 0, prefix = '', suffix = '', decimals = 1, length = 100, fill = ' '):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        mode        - Optional  : Mode of output (Int : (dafault 0 : grahical progress bar , 1 : standard text))
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    if mode == 0 :
        filledLength = int(length * iteration // total)
        bar = fill * filledLength + '-' * (length - filledLength)
        print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\r')
        # Print New Line on Complete
        if iteration == total:
            print()
    elif mode == 1:
        if current_percent != round(float(percent),1):
            current_percent = round(float(percent),1)
            print(percent+"%")
        return current_percent
    return 0
!pip install pytesseract
stemmer = PorterStemmer()


processed_docs = documents['headline_text'].map(preprocess)
processed_docs
dictionary = gensim.corpora.Dictionary(processed_docs)
count = 0
for k, v in dictionary.iteritems():
    print(k, v)
    count += 1
    if count > 10:
        break
dictionary
# ตัดคำฟุ่มเฟือยออก โดยลบคำที่ปรากฏบ่อยเกิน 50% และให้เหลือไปถึง1พันคำ
dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=1000)  #reduce to 1K word


# Gensim doc2bow
# For each document we create a dictionary reporting how many
# words and how many times those words appear. Save this to ‘bow_corpus’, then check our selected document earlier.
bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]

from gensim import corpora, models
tfidf = models.TfidfModel(bow_corpus)
corpus_tfidf = tfidf[bow_corpus]
from pprint import pprint
for doc in corpus_tfidf:
    pprint(doc)
    break

# run LDA แบ่งคำออกเป็น9กลุ่ม โดยใช้ cpu ทุกตัว (serverที่เราใช้มี32cpu)
#worker = cpu core amount -1
lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=10, id2word=dictionary, passes=2, workers=31)

# For each topic, we will explore the words occuring in that topic and its relative weight.
for idx, topic in lda_model.print_topics(-1):
    print('Topic: {} \nWords: {}'.format(idx, topic))
unseen_document = 'เป็นเบาหวานจะมีผลกระทบกับการตั้งครรภ์'
bow_vector = dictionary.doc2bow(preprocess(unseen_document))
for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):
    print("Score: {}\t Topic: {}".format(score, lda_model.print_topic(index, 15)))
dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)

# Dictionary for each document
bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]
# print(bow_corpus[4310])

# Applying TF-IDF
tfidf = models.TfidfModel(bow_corpus)
corpus_tfidf = tfidf[bow_corpus]
# for doc in corpus_tfidf:
#     pprint(doc)
#     break

# Train corpus with LDA
lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=10, id2word=dictionary,
                                       passes=2, workers=2)
# for idx, topic in lda_model.print_topics(-1):
#     print('Topic: {} \nWords: {}'.format(idx, topic))

# LDA with TF-IDF
lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=10, id2word=dictionary,
                                             passes=2, workers=4)
for idx, topic in lda_model_tfidf.print_topics(-1):
    print('Topic: {} Word: {}'.format(idx, topic))

# Performance evaluation of LDA with Bag of Words
# for index, score in sorted(lda_model[bow_corpus[4310]], key=lambda tup: -1*tup[1]):
#     print("\nScore: {}\t \nTopic: {}".format(score, lda_model.print_topic(index, 10)))

# Performance Evaluation of LDA with TF-IDF
for index, score in sorted(lda_model_tfidf[bow_corpus[4310]], key=lambda tup: -1*tup[1]):
    print("\nScore: {}\t \nTopic: {}".format(score, lda_model_tfidf.print_topic(index, 10)))

# Testing
unseen_document = 'จะท้องไหมคะ'
bow_vector = dictionary.doc2bow(preprocess(unseen_document))
for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):
    print("Score: {}\t Topic: {}".format(score, lda_model.print_topic(index, 5)))
# Print the keyword of topics
pprint(lda_model.print_topics())
doc_lda = lda_model[bow_corpus]<jupyter_output>[(0,
  '0.403*"กำเนิด" + 0.114*"สอดใส่" + 0.076*"ใช้จ่าย" + 0.067*"เพศสัมพันธ์" + '
  '0.026*"สัมพันธ์" + 0.021*"กะปริบกะปรอย" + 0.018*"สิทธิ์" + 0.017*"ฉุกเฉิน" '
  '+ 0.016*"ประกัน" + 0.015*"ป้องกัน"'),
 (1,
  '0.250*"สัมพันธ์" + 0.128*"เพศสัมพันธ์" + 0.106*"ป้องกัน" + 0.076*"ฉุกเฉิน" '
  '+ 0.047*"เพศสัมพัน" + 0.045*"หลั่งนอก" + 0.027*"เครียด" + 0.026*"คัดเต้า" + '
  '0.018*"ตั้งท้อง" + 0.018*"หน้าอก"'),
 (2,
  '0.273*"อันตราย" + 0.126*"รับประทาน" + 0.058*"อักเสบ" + 0.043*"เพศสัมพันธ์" '
  '+ 0.039*"ป้องกัน" + 0.035*"ท้องน้อย" + 0.027*"มะเร็ง" + 0.023*"เพสสัมพัน" + '
  '0.022*"ข้อมูล" + 0.022*"อารมณ์"'),
 (3,
  '0.154*"ฮอร์โมน" + 0.100*"เครียด" + 0.085*"เพศสัมพันธ์" + 0.082*"อาเจียน" + '
  '0.080*"สิทธิ์" + 0.049*"ตั้งท้อง" + 0.049*"เมอซิลอน" + 0.041*"สุขภาพ" + '
  '0.033*"เหนื่อย" + 0.026*"ตั้งใจ"'),
 (4,
  '0.323*"เพศสัมพันธ์" + 0.116*"ฉุกเฉิน" + 0.106*"น้ำตาล" + 0.057*"ผ่าตัด" + '
  '0.033*"แต่งงาน" + 0.031*"กำเนิด" + 0.021*"เคลื่อน" + 0.020*"หายขาด" + '
  '0.020*"อนุญาต" + 0.019[...]<jupyter_text>**============================================================================**<jupyter_code>!pip3 install spacy
!python3 -m spacy download en #Language model
!pip3 install gensim # For topic modeling
!pip3 install pyLDAvis # For visualizing topic models
import re
import numpy as np
import pandas as  pd
from pprint import pprint# Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel# spaCy for preprocessing
import spacy# Plotting tools
import pyLDAvis
import pyLDAvis.gensim
import matplotlib.pyplot as plt
%matplotlib inline

# Compute Perplexity
print('\nPerplexity: ', lda_model.log_perplexity(bow_corpus))  
# a measure of how good the model is. lower the better.

# Compute Coherence Score
coherence_model_lda = CoherenceModel(model=lda_model, texts=processed_docs, dictionary=dictionary, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)<jupyter_output>Perplexity:  -4.447146089283022

Coherence Score:  0.5623758434479108<jupyter_text>Output:
- Lower the perplexity better the model.
- Higher the topic coherence, the topic is more human interpretable.<jupyter_code># Visualize the topics
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, bow_corpus, dictionary)
vis<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15953332896375436, 'fraction_numerical': 0.024775512879334077, 'mean_word_length': 4.333100314575323, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '108424', 'license': 'MIT', 'n_tokens_mistral': 6271, 'n_tokens_neox': 6046, 'n_words': 1116, 'path': 'topic_modeling_th.ipynb', 'repo_name': 'thopiyawan/DEMO_rami_dialogflow', 'size': '15257'}