<jupyter_start><jupyter_text>4.1.4 Logistic Regression Example with TF-IDF  TF-IDF Feature Example<jupyter_code>import os

import pandas as pd
import numpy as np

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
DATA_IN_PATH = './data_in/' 
DATA_OUT_PATH = './data_out/'
TRAIN_CLEAN_DATA = 'train_clean.csv'

RANDOM_SEED = 42
TEST_SPLIT = 0.2
train_data = pd.read_csv( DATA_IN_PATH + TRAIN_CLEAN_DATA )
reviews = list(train_data['review'])
sentiments = list(train_data['sentiment'])
vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="char", sublinear_tf=True, ngram_range=(1,3), max_features=5000) 

X = vectorizer.fit_transform(reviews)
y = np.array(sentiments)
X
features = vectorizer.get_feature_names()
X_train, X_eval, y_train, y_eval = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RANDOM_SEED)
lgs = LogisticRegression(class_weight='balanced') 
lgs.fit(X_train, y_train) 
predicted = lgs.predict(X_eval)
print("Accuracy: %f" % lgs.score(X_eval, y_eval))
TEST_CLEAN_DATA = 'test_clean.csv'

test_data = pd.read_csv(DATA_IN_PATH + TEST_CLEAN_DATA)
testDataVecs = vectorizer.transform(test_data['review'])
test_predicted = lgs.predict(testDataVecs)
print(test_predicted)
if not os.path.exists(DATA_OUT_PATH):
    os.makedirs(DATA_OUT_PATH)

answer_dataset = pd.DataFrame({'id': test_data['id'], 'sentiment': test_predicted})
answer_dataset.to_csv(DATA_OUT_PATH + 'lgs_tfidf_answer.csv', index=False, quoting=3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11499364675984752, 'fraction_numerical': 0.010165184243964422, 'mean_word_length': 7.848314606741573, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '365135', 'license': 'Apache-2.0', 'n_tokens_mistral': 612, 'n_tokens_neox': 598, 'n_words': 103, 'path': '4.TEXT_CLASSIFICATION/4.1.4 Linear Regression with TF-IDF.ipynb', 'repo_name': 'minalang/tensorflow-deeplearning', 'size': '1574'}