<jupyter_start><jupyter_text>Seoul Bike Text Mining - LDA Analyse using Okt Objects- 구글 플레이 스토어 '따릉이' 어플 리뷰와 ‘서울 자전거 따릉이’ 공식 웹사이트 내에 시민의견수렴 게시판에 작성된 의견 텍스트를 수집- 오프라인의 이용 경험을 구분 지어 분석하기 위해 앱 리뷰를 온라인 사용 경험 분석- 시민의견수렴 게시판을 오프라인 사용 경험 분석으로 활용- LDA 토픽 모델링 통해 텍스트 속에 이용 경험의 문제점을 온라인과 오프라인으로 구별하는 분석을 진행<jupyter_code>from collections import Counter
import pandas as pd
import numpy as np
from konlpy.tag import Mecab
from konlpy.tag import Okt
from ckonlpy.tag import Twitter

ctwitter = Twitter()
mecab = Mecab()
twitter = Okt()<jupyter_output><empty_output><jupyter_text>전처리<jupyter_code>from ckonlpy.tag import Postprocessor #전처리 라이브러리
from ckonlpy.tag import Twitter
import warnings
warnings.simplefilter("ignore")

ctwitter = Twitter()
ctwitter.add_dictionary(['따릉이','자출',
                       '대여소','연결거치','거치대','고객센터',
                        '다시','퇴근시간','대여시간',
                        '출근시간','한시간','두시간','임시폐쇄',
                         '자동로그인','UX', '1일권', '일일권','가성비',
                         '상담원','재로그인', '인증번호', '가입','렉', '랙','제로페이','결제','큐알코드','QR','뒤로가기',
                         '대여번호''결제','결재','여러번','불편','개불편함','내세금','전반적', '지도화면',
                        ], 'Noun')
#조사
ctwitter.add_dictionary(['까지','하냐'], 'Josa')
#형용사
ctwitter.add_dictionary(['레알','허접한','오지게' ,'어려워','안되', '열받','안됨', '안와' ,'참어렵네'], 'Adjective')
#부사

ctwitter.add_dictionary(['왜', '좀','진짜','최악' , '참'], 'Adverb', force=True)

ctwitter.add_dictionary(['타고','요청드립니다','부탁드립니다',
                         '요청 드립니다','부탁 드립니다','하기'],'Verb')

#무의미한 형태소 및 단어 제거
stopwords = {'수가','까지','드립','번','수','것', '앞','해주','왜',
             '곳','떄','좀','어요','요','이','감사','이','그','고','제','쪽','더','후','시','거',
'려고','적','저','데','등','역','중','해주시','내','면서','어서','때','뭐','못','마다','더니','해도','해도','다가','어가','개','하라','하나','걸'}


#단어 대치
replace = {'어플리케이션': '어플',
           '앱': '어플',
           '폐쇠':'페쇄',
           '젛은':'좋은',
           '연계':'연동',
           '맵':'지도',
           '인터페이스':'UI',
           '유아이':'UI',
           'ui': 'UI',
           'T머니' : '티머니',
           '렉':'오류',
           '랙':'오류',
           '가입': '회원가입',
          '결재':'결제',
           '버젼':'버전',
           '고객 센터': '고객센터',
           '에러': '오류',
           '버그': '오류',
           '업뎃':'업데이트' ,'업댓':'업데이트','업디이트':'업데이트','리뉴얼':'업데이트','개편':'업데이트','업그레이드':'업데이트',
           '업데이틀':'업데이트',
           '업글' :'업데이트',
           '비번':'비밀번호',
           '머지':'뭐지',
           '자동 로그인':'자동로그인',
           '딜레이':'지연',
           '정거장': '대여소','정류장': '대여소','대여수': '대여소',
           '큐알':'QR','qr':'QR','큐알코드':'QR',
           '알코':'QR',
           '리붓':'재부팅',
           '튜토리얼':'이용설명',
           '이용안내':'이용설명',
           '안되요':'안돼요',
           '카톡': '카카오톡',
           '먹통' :'오류',
           '제하': '결제',
            '카카오': '카카오톡',
            '카카오계정': '카카오톡',
           ('안되고', 'Adjective'): '안되',
           '않':'안되',
           '마니':'많이',
           '허다': '많다',
           '안와': '안되',
           '안': '안되',
           '폰':'휴대폰',
           '지도화면':'지도',
           '안됨':'안되',
           '팅김':'오류',
           '장애':'오류',
           '내세금':'세금',
           '개불편함':'불편',
           '연결거치': '연결'}

postprocessor = Postprocessor(ctwitter, stopwords = stopwords,replace = replace)<jupyter_output><empty_output><jupyter_text>Google Play Store  데이터 확인<jupyter_code>df = pd.read_csv('sbike_google_sept copy.csv').drop('Unnamed: 0', axis=1)

reviews = df['review_text']
df.head(3)
# 별점 4점 이상 준 유저수 -> 매우 적음
df[df['rating'] >=4].count()<jupyter_output><empty_output><jupyter_text>사전 추가기존 형태소 분석기에 추가되지 않은 단어 들을 추가<jupyter_code>ctwitter.add_dictionary(['따릉이','자출',
                       '대여소','거치대','고객센터',
                       '임시폐쇄',
                         '자동로그인','UX', '1일권', '일일권','가성비',
                         '상담원','재로그인', '인증번호', '가입','렉', '랙','제로페이','결제','큐알코드','뒤로가기',
                         '대여번호''결제','결재','여러번','불편','개불편함','내세금','전반적', '지도화면','화',
                        ], 'Noun')<jupyter_output><empty_output><jupyter_text>형태소 분석 진행<jupyter_code>def twitterWithStemming(text):
    sentences_tag = []
    for sentence in text:
        morph = twitter.pos(sentence, stem=False )
        sentences_tag.append(morph)
    return sentences_tag
def stop_wording(sentences_tag):
    token = []
    token_set={}
    token_table=[]
    cnt = 0
    for row in sentences_tag:
        token=[]

        for word,tag in row:

            if tag in 'Noun':
                token.append(word)
            elif tag in 'Adjective':
                token.append(word)
            elif tag in 'Adverv':
                token.append(word)
            elif tag in 'VerbPrefix':
                token.append(word)
                
            elif tag in 'Verb':
                token.append(word)


        token_table.append(token)
    return token_table<jupyter_output><empty_output><jupyter_text>분석처리를 위한 파일 합병 엑셀화<jupyter_code>df_text = twitterWithStemming(reviews)
b = stop_wording(df_text)<jupyter_output><empty_output><jupyter_text>LDA Analysis- Gensim Package 사용- 형태소 분석기 : Okt- 전체 리뷰, 부정리뷰를 나누어서 분석 진행- 긍정 리뷰의 경우 별점 4~5점은 준 리뷰의 갯수가 매우 적어 분석을 진행 하지 않음<jupyter_code>from gensim import corpora

dictionary  = corpora.Dictionary(b)
corpus = [dictionary.doc2bow(text) for text in b ]<jupyter_output><empty_output><jupyter_text>전체 리뷰에 대한 LDA 분석<jupyter_code>import gensim
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=7, alpha='auto', 
                                      update_every=1, chunksize=20000, passes=10)
lda.print_topics()
from gensim.models.coherencemodel import CoherenceModel
# Compute Perplexity
print('\nPerplexity: ', lda.log_perplexity(corpus)) # a measure of how good the model is. lower the better.
 
# Compute Coherence Score
coherence_model_lda = CoherenceModel(model=lda, texts=b, dictionary=dictionary, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)<jupyter_output>Perplexity:  -7.3276142666133754

Coherence Score:  0.39175704106359943<jupyter_text><jupyter_code>#구글 플레이 스토어 7개 토픽 모델링 저장
from gensim.test.utils import datapath

temp_file = 'LDA/google_num7'
lda.save(temp_file)

lda_google7 = gensim.models.ldamodel.LdaModel.load(temp_file)
lda_google7.print_topics()
for d in b:
    bow = dictionary.doc2bow(d)
    t = lda.get_document_topics(bow)
t # 토픽 출현 확률
import numpy as np
topics_terms = lda.state.get_lambda()
topics_terms_proba = np.apply_along_axis(lambda x: x/x.sum(),1,topics_terms)
probality=[]
for i in range(5):
    
    probality.append(topics_terms[i].sum()/topics_terms.sum())
topics = lda.print_topics()
def get_topics(topic):
    listed = topic.split('+')
    topics=[]
    for i in listed:
        topics.append(i[i.index('*')+1:].replace('"', '').rstrip())
    
    return topics
def get_topics_number(topic):
    listed = topic.split('+')
    topics=[]
    for i in listed:
        topics.append(i[:i.index('*')].replace('"', '').rstrip())
    
    return topics
topics_list=[]

for i in topics:
    topics_list.append(get_topics(i[1]))
topics_list
probality
topics = lda.print_topics()
def get_topics_number(topic):
    listed = topic.split('+')
    topics=[]
    for i in listed:
        topics.append(i[:i.index('*')].replace('"', '').rstrip())
    
    return topics<jupyter_output><empty_output><jupyter_text>상위 토픽 10개 항목에 대한 토픽 분포<jupyter_code>topics_num_list=[]

for i in topics:
    topics_num_list.append(get_topics_number(i[1]))
# print(topics_num_list)
# pd.DataFrame(topics_num_list).to_excel('Yoon/citizen_lda_num.xlsx')
pd.DataFrame(topics_num_list)
topics_num_list=[]

for i in topics:
    topics_num_list.append(get_topics_number(i[1]))
pd.DataFrame(topics_num_list).to_excel('Yoon/google_lda_num.xlsx')
import pandas as pd
pd.DataFrame(topics_list).append(probality).to_excel('Yoon/google_lda_refined.xlsx')<jupyter_output><empty_output><jupyter_text>부정적 점수를 준 리뷰<jupyter_code>df = pd.read_csv('sbike_google_sept.csv')
bad_reviews = df[df['rating']<3]['review_text']
sentence_tag =do_twitter(bad_reviews)
bad_tokenized = stop_wording(sentence_tag)
dictionary  = corpora.Dictionary(bad_tokenized)
corpus = [dictionary.doc2bow(text) for text in bad_tokenized ]
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, update_every=1, chunksize=10000, passes=20)
lda.print_topics()<jupyter_output><empty_output><jupyter_text>시민의견수렴 게시판<jupyter_code>df = pd.read_excel('Seoulbike_complain.xlsx')

text = df['content']
str_text = []
for row in text:
    str_text.append(str(row))
sentence_tag =do_twitter(str_text)
citizen_tokenized = stop_wording(sentence_tag)
dictionary  = corpora.Dictionary(citizen_tokenized)
corpus = [dictionary.doc2bow(text) for text in citizen_tokenized ]
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, update_every=1, chunksize=10000, passes=20)
lda.print_topics()
lda_result = lda.print_topics()
list(lda_result[1])[1:]
verb_list = []
for sentence in sentences_tag:
    for word, tag in sentence:
        if tag in ['Verb']:
            
            verb_list.append(word)
            
len(adj_list)
counts_adj = Counter(verb_list)
print(counts_adj.most_common(100))
counts_adj = Counter(adj_list)
print(counts_adj.most_common(100))
import pandas as pd
df = pd.DataFrame(counts_adj.most_common(100))
df.to_excel('Yoon/s_bike_google_adj.xlsx')<jupyter_output><empty_output><jupyter_text>명사 추출후 상위 10개 항목 정렬<jupyter_code>df['twitter(no Stem)'] = df['review_text'].apply(lambda x: twitter.pos(x))
df['twitter(Stem)'] = df['review_text'].apply(lambda x: twitter.pos(x, stem=True))
df['cutomizedTwitter(no Stem)'] = df['review_text'].apply(lambda x: ctwitter.pos(x))
df['cutomizedTwitter(Stem)'] = df['review_text'].apply(lambda x: ctwitter.pos(x, stem=True))
df.head(3)
df.to_excel('2020_seoulbike/comparsion_twitter.xlsx')
import pandas as pd

df = pd.DataFrame.from_dict(n_counts, orient='index').reset_index()
df = df.rename(columns={'index':'noun', 0:'count'})
df=df.sort_values(by=['count'], axis=0, ascending=False)

#엑셀로 저장
df.to_excel("Yoon/sbike_google_noun.xlsx")
import pandas as pd
df2 = pd.DataFrame.from_dict(counts_adj, orient='index').reset_index()
df2 = df2.rename(columns={'index':'noun', 0:'count'})
df2 = df2.sort_values(by=['count'], axis=0, ascending=False)
df2.head()
df2.to_excel('./google_adj_count.xlsx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17335008213353947, 'fraction_numerical': 0.01178857860662866, 'mean_word_length': 3.4689119170984455, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '253290', 'license': 'MIT', 'n_tokens_mistral': 4851, 'n_tokens_neox': 5564, 'n_words': 694, 'path': 'notebook/lda_anysis_twitter.ipynb', 'repo_name': 'metterian/seoulbike-process-mining', 'size': '10349'}