<jupyter_start><jupyter_text>HW8 HW7 V4<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.tokenize.casual import casual_tokenize
tfidf_model = TfidfVectorizer(tokenizer = casual_tokenize)
from sklearn.preprocessing import MinMaxScaler
from collections import Counter
import pandas as pd
train=pd.read_csv("../WK7/kaggle-sentiment/train.tsv", delimiter='\t')
y=train['Sentiment'].values
X=train['Phrase'].values
train.to_csv('kaggle_csv.csv')

df = pd.DataFrame()
df['Phrase'] = train['Phrase']
df['S0'] = train['Sentiment']

def get_lda(df):
    tfidf_docs = tfidf_model.fit_transform(raw_documents = df['Phrase']).toarray()
    mask = df.PoN.astype(bool).values
    n_centroid = tfidf_docs[mask].mean(axis=0)
    p_centroid = tfidf_docs[~mask].mean(axis=0)
    negativity_score = tfidf_docs.dot(n_centroid - p_centroid)
    df['lda_score'] = MinMaxScaler().fit_transform(negativity_score.reshape(-1,1))
    df['lda_predict'] = (df.lda_score > .5).astype(int)
    score = (1. - (df.PoN - df.lda_predict).abs().sum() / len(df)).round(3) 
    return score, df, negativity_score, n_centroid, p_centroid

def get_lda_submission(df, n_centroid, p_centroid):
    tfidf_docs = tfidf_model.fit_transform(raw_documents = df['Phrase']).toarray()
    negativity_score = tfidf_docs.dot(n_centroid - p_centroid)
    df['lda_score'] = MinMaxScaler().fit_transform(negativity_score.reshape(-1,1))
    df['lda_predict'] = (df.lda_score > .5).astype(int)
    return df

df['PoN'] = [0 if x < 2 else 1 for x in df['S0']]
len(df[df['PoN'] == 1])
lda_score, df, neg_score, n_cent, p_cent = get_lda(df)

columns = ['Phrase']
new_df =  pd.DataFrame(df, columns = columns)
predicted_df = get_lda_submission(new_df, n_cent, p_cent)

predicted_df['actual'] = df['S0']
predicted_df['bool'] = [0 if x < 2 else 1 for x in predicted_df['actual']]
predicted_df['check'] = predicted_df.apply(lambda x: (x['lda_predict'] == x['bool']), axis=1)

Counter(predicted_df['check'])
predicted_df_test = get_lda_submission(test, n_cent, p_cent)
print(len(n_cent))
print(len(p_cent))
print(len(test))
test=pd.read_csv("../WK7/kaggle-sentiment/test.tsv", delimiter='\t')
test

training_encoded = pd.get_dummies(train, columns=['Phrase'])
test_encoded = pd.get_dummies(test, columns=['Phrase'])
test_encoded_for_model = test_encoded.reindex(columns = training_encoded.columns, 
    fill_value=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15734989648033126, 'fraction_numerical': 0.011180124223602485, 'mean_word_length': 6.2119402985074625, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '229992', 'license': 'MIT', 'n_tokens_mistral': 978, 'n_tokens_neox': 954, 'n_words': 152, 'path': 'assets/all_html/2019_11_30_HW8_HW7_v4.ipynb', 'repo_name': 'dskw1/dskw1.github.io', 'size': '2415'}