<commit_before>import json
import os
from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer
from sklearn.model_selection import train_test_split


file_name = os.path.join(os.path.dirname(__file__), '../../data/processed/livedoor_tokenized.json')
with open(file_name) as f:
    livedoor = json.load(f)

docs = livedoor['data']
docs = [' '.join(d) for d in docs]
X_train, X_test, y_train, y_test = train_test_split(docs, livedoor['label'], test_size=0.4, random_state=0)

#tfidf_transformer = TfidfTransformer()
#X_train_tfidf = tfidf_transformer.fit_transform(texts)
vectorizer = TfidfVectorizer()
X_train = vectorizer.fit_transform(docs)
print(X_train.shape)
print(y_train.shape)
from sklearn.naive_bayes import MultinomialNB
text_clf = MultinomialNB().fit(X_train, y_train)
print(text_clf.score(X_test, y_test))
<commit_msg>implement BoW and TF-IDF<commit_after># -*- coding:utf-8 -*-
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer


corpus = [
	'This is a pen.',
	'That is a bot.',
	'These are red document and blue document.',
]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)
print(vectorizer.get_feature_names())
print(X.toarray())

vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(corpus)
print(vectorizer.get_feature_names())
print(X.toarray())
{'fraction_non_alphanumeric': 0.12341062079281974, 'fraction_numerical': 0.0029917726252804786, 'mean_word_length': 6.734104046242774, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5903611', 'n_tokens_mistral': 492, 'n_tokens_neox': 483, 'n_words': 95}