<jupyter_start><jupyter_text>형태소 분석 by konlpy<jupyter_code>pos_tagger = Twitter()
def tokenize(doc):
    return ['/'.join(t) if t[1] in ['Noun', 'Verb', 'Adjective', 'Modifier', 'Adverb'] for t in pos_tagger.pos(doc, norm=True, stem=True)]
review_docs = [tokenize(review) for review in reviews]
review_docs[1]
tokens = [t for d in review_docs for t in d]
len(tokens)
# 저장
import pickle
with open("../dataset/tokens.pickle", 'wb') as f:
    pickle.dump(tokens, f)
# 로드
f = open('../dataset/tokens.pickle', 'rb')
tokens = pickle.load(f)<jupyter_output><empty_output><jupyter_text>분석에 유의미한 품사들만 선별 --> 명사(Noun), 동사(Verb), 형용사(Adjective), 관형사(Modifier), 부사(Adverb)<jupyter_code>def tokenize_selected(doc):
    return ['/'.join(t) for t in pos_tagger.pos(doc, norm=True, stem=True) if t[1] in ['Noun', 'Verb', 'Adjective', 'Modifier', 'Adverb']]
review_docs2 = [tokenize_selected(review) for review in reviews]
tokens_selected = [t for d in review_docs2 for t in d]
len(tokens_selected)
with open("../dataset/tokens_selected.pickle", 'wb') as f:
    pickle.dump(tokens_selected, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16726943942133815, 'fraction_numerical': 0.0045207956600361665, 'mean_word_length': 6.006329113924051, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '193184', 'license': 'MIT', 'n_tokens_mistral': 453, 'n_tokens_neox': 486, 'n_words': 106, 'path': '02_preprocessing/03_data_preprocessing.ipynb', 'repo_name': 'yunah0515/dss7_Personal_Project', 'size': '1106'}