<jupyter_start><jupyter_text>1. 특수문자 제거, 문장 분리, 띄어쓰기는 공통적으로 적용<jupyter_code># 특수문자 제거
import re

# 간편한 버전(예지버전) 사용
def cleanse(text):
    pattern = re.compile(r'\s+')
    text = re.sub(pattern, ' ', text)
    text = re.sub('[^가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9]', ' ', text)
    return text

train['comments'] = train['comments'].apply(cleanse)
train.iloc[100:105]
# 띄어쓰기
from pykospacing import spacing
train['comments'] = train['comments'].apply(spacing)
train.iloc[100:105]
# 문장 분리
import kss
train['comments'] = train['comments'].apply(kss.split_sentences)
train['comments'] = [','.join(map(str, ls)) for ls in train['comments']]
train.iloc[100:105]
# 중복 제거
from soynlp.normalizer import *
train['comments'] = [repeat_normalize(comment, num_repeats=2) for comment in train['comments']]
train.iloc[100:105]<jupyter_output><empty_output><jupyter_text>2. 각 형태소 분석기별 데이터 생성<jupyter_code># 데이터프레임 생성
khai_train = pd.DataFrame(columns=['words', 'postagged'])
mecab_train = pd.DataFrame(columns=['words', 'postagged'])
okt_train = pd.DataFrame(columns=['words', 'postagged'])
# khaiii
from khaiii import KhaiiiApi
kh = KhaiiiApi()

words = []
postagged = []
for comment in train['comments']:
    # 문장 -> 형태소 객체로 분리
    sentence = kh.analyze(comment)
    # 형태소 -> 한 문장으로 다시 합쳐서 저장
    words.append(' '.join([morph.lex for morph in sentence]))
    # 품사 태깅
    #postagged.append(' '.join([morph.morphs.tag for morph in sentence]))

khai_train['words'] = words
from sklearn.naive_bayes             import MultinomialNB
from sklearn.ensemble                import RandomForestClassifier
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model            import LogisticRegression
from sklearn.model_selection         import GridSearchCV, train_test_split, cross_val_score
from sklearn.metrics                 import accuracy_score, recall_score, roc_auc_score, f1_score
from sklearn.pipeline                import Pipeline
from sklearn.svm                     import SVC
from sklearn.tree                    import DecisionTreeClassifier
from lightgbm                        import LGBMClassifier
from konlpy.tag import Okt
t = Okt()
def t_tokenizer(text):
    tokens_ko = t.morphs(text)
    return tokens_ko
models = [RandomForestClassifier(), LogisticRegression(), SVC(), LGBMClassifier()]

def get_score(model):
    # 훈련: train 전체 / 테스트: dev 전체
    X_train = khai_train['words']
    X_test = test['comments']
    y_train = train['label']
    y_test = test['label']
    
    # Setting up the pipeline
    vec_pipe = Pipeline([
                        ("vec", TfidfVectorizer(tokenizer=t_tokenizer)), 
                        ("model", model)
                        ])
    
    # Setting the VEC hyperparameters
    vec_pipe_params = {"vec__ngram_range" : [(1,2)], 
                       "vec__stop_words"  : [None],
                       "vec__min_df" : [3],
                       "vec__max_df" : [0.9]}
    
    
    # Instantiating the grid search
    vec_gs = GridSearchCV(vec_pipe,
                          param_grid=vec_pipe_params,
                          cv=3)

    # Fitting the model to the training data
    vec_gs.fit(X_train, y_train);
    
    # Predicting
    train_pred = vec_gs.predict(X_train)
    test_pred = vec_gs.predict(X_test)

    # Score
    f1 = [f1_score(train_pred, y_train, average='macro'),
          f1_score(test_pred, y_test, average='macro')]
    acc = [accuracy_score(y_train, train_pred),
           accuracy_score(y_test, test_pred)]
    score_df = pd.DataFrame(data={'F1 Score' : f1, 'Accuracy' : acc},
                            index=['Train', 'Test'])
    return score_df
for model in models:
    print("Model : {}".format(model))
    print(get_score(model), '\n', '------------------------------', sep='')<jupyter_output>Model : RandomForestClassifier()
       F1 Score  Accuracy
Train  0.998677  0.998734
Test   0.487739  0.513800
------------------------------
Model : LogisticRegression()
       F1 Score  Accuracy
Train  0.832283  0.841565
Test   0.579778  0.585987
------------------------------
Model : SVC()
       F1 Score  Accuracy
Train  0.973995  0.974924
Test   0.536260  0.552017
------------------------------
Model : LGBMClassifier()
       F1 Score  Accuracy
Train  0.787965  0.796479
Test   0.546274  0.552017
------------------------------
{'fraction_non_alphanumeric': 0.1488576044311101, 'fraction_numerical': 0.03623355642741749, 'mean_word_length': 2.543744889615699, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '140367', 'license': 'MIT', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1631, 'n_words': 321, 'path': 'code/Jaecheol/.ipynb_checkpoints/02_2_Model_Comparison-checkpoint.ipynb', 'repo_name': 'kkobooc/NLP_KoreanHateSpeech', 'size': '4333'}