<jupyter_start><jupyter_text>Tokenize<jupyter_code>import numpy as np
from konlpy.tag import *
from sklearn.feature_extraction.text import CountVectorizer
posToUse=["NNP","NNG","MAG","NP","VV","VV+EF",'XSV+EC']

def getTokens(s):
    global posToUse
    return " ".joi

n([ i[0] for i in  Mecab().pos(s) if i[1] in posToUse ] )<jupyter_output><empty_output><jupyter_text>1. Simple Version by NLTK<jupyter_code>from nltk.tokenize import word_tokenize  
s="나는 너를 사랑한다"

print(word_tokenize("나는 나를 사랑한다"))  
print(word_tokenize(getTokens(s)))<jupyter_output>['나는', '나를', '사랑한다']
['나', '너', '사랑', '한다']<jupyter_text>2. Keras Tokenizer<jupyter_code>corpus = [
    "고인물은 게임을 즐긴다.",
    "남중호는 게임을 잘한다.",
    "남중구는 게임을 못한다.",
    "나는 나란 놈 사랑한다.18",
    "너의 게임 나의 게임 사랑하라"
]
s=" ".join(corpus)
print('All string :' ,s)
from tensorflow.keras.preprocessing.text import Tokenizer
tokenizer = Tokenizer()
sentences=[getTokens(r) for r in corpus]
tokenizer.fit_on_texts (sentences)
print(tokenizer.texts_to_sequences(sentences))
tokenizer.word_index<jupyter_output><empty_output><jupyter_text>3. Keras pad_sequence<jupyter_code>from tensorflow.keras.preprocessing.sequence import pad_sequences
sentences=[getTokens(r) for r in corpus]
encoded = tokenizer.texts_to_sequences(sentences)
pad_sequences(encoded)
pad_sequences(encoded,padding = 'post')<jupyter_output><empty_output><jupyter_text>cf) Sentence Tokenizer<jupyter_code>from nltk.tokenize import sent_tokenize
text="Mr. Nam은 게임을 즐긴다. 남중호는 게임을 잘한다...그런가 보다."
print(sent_tokenize(text))
?tokenizer.texts_to_sequences<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13862633900441085, 'fraction_numerical': 0.004410838059231254, 'mean_word_length': 7.060913705583756, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '99709', 'license': 'Apache-2.0', 'n_tokens_mistral': 712, 'n_tokens_neox': 756, 'n_words': 119, 'path': '7-1.NLP/6.Tockenizer.ipynb', 'repo_name': 'dnworry/AICentroPreLab', 'size': '1587'}