<jupyter_start><jupyter_text>Spam  Persistent homology<jupyter_code>flatten = lambda l: [item for sublist in l for item in sublist]
train = pd.read_csv('data/train.csv')
test = pd.read_csv('data/test.csv')
texts = pd.read_csv('data/texts.csv')
texts.TEXT = texts.TEXT.str.replace('[^\w\s]','')

w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()

def lemmatize_text(text):
    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]

texts['TEXT_L'] = texts.TEXT.apply(lemmatize_text)
texts_l = texts.TEXT_L

texts_ph = []

for element in texts_l:
    try:
        a = PersistentNonTrained(split_sent=element, min_count=1, workers=-1)
        dgms = a.persistent()
        texts_ph.append(flatten([[p.birth for p in dgms[1]], [p.death for p in dgms[1]]]))
    except:
        texts_ph.append([])
text_ph_df = pd.DataFrame(texts_ph)
text_ph_df.head(10)
text_ph_df['ID'] = texts['ID']

X = pd.merge(train, text_ph_df.fillna(0), on='ID', how='left').drop(['ID', 'CATEGORY'], 1).values
y = pd.factorize(train['CATEGORY'])[0]
X.shape
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
X_test.shape
logist = LogisticRegression()
def get_roc_auc(model, X_test, y_test):
    fpr, tpr, _ = roc_curve(y_test, model.predict_proba(X_test)[:,1], pos_label=1)
    auc_s = auc(fpr, tpr)
    
    return fpr, tpr, auc_s 
logist.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Tf-Idf<jupyter_code>word_vectorizer = TfidfVectorizer(
    sublinear_tf=True,
    strip_accents='unicode',
    analyzer='word',
    token_pattern=r'\w{1,}',
    stop_words='english',
    ngram_range=(1, 2),
    max_features=10000)

text_tfidf = word_vectorizer.fit_transform(texts.TEXT).toarray()

text_tfidf = pd.DataFrame(text_tfidf)

text_tfidf['ID'] = texts['ID']
X_ti = pd.merge(train, text_tfidf, on='ID', how='left').drop(['ID', 'CATEGORY'], 1).values
y_ti = pd.factorize(train['CATEGORY'])[0]
X_train_ti, X_test_ti, y_train_ti, y_test_ti = train_test_split(X_ti, y_ti, test_size=0.3)
logist_ti = LogisticRegression()
logist_ti.fit(X_train_ti, y_train_ti)<jupyter_output><empty_output><jupyter_text>BoW<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer

bigram_vectorizer = CountVectorizer(ngram_range=(1, 2), token_pattern=r'\b\w+\b', min_df=1)

text_bow = bigram_vectorizer.fit_transform(texts.TEXT).toarray()

text_bow = pd.DataFrame(text_bow)

text_bow['ID'] = texts['ID']

X_bow = pd.merge(train, text_bow, on='ID', how='left').drop(['ID', 'CATEGORY'], 1).values
y_bow = pd.factorize(train['CATEGORY'])[0]
X_train_bow, X_test_bow, y_train_bow, y_test_bow = train_test_split(X_bow, y_bow, test_size=0.3)
from sklearn.svm import SVC
svc = SVC(probability=True, verbose=True)
svc.fit(X_train_bow, y_train_bow)
plt.figure(figsize=(10, 7))

lw = 2

fpr, tpr, auc_s = get_roc_auc(logist, X_test, y_test)

plt.plot(fpr, tpr, color='darkorange',
         lw=lw, 
         label=('Logitstic regression + w2v + PH (area under the curve:{0:0.2f})'.format(auc_s)))

fpr, tpr, auc_s = get_roc_auc(logist_ti, X_test_ti, y_test_ti)

plt.plot(fpr, tpr, color='red',
         lw=lw, 
         label=('Logitstic regression + tf-idf (area under the curve:{0:0.2f})'.format(auc_s)))

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])

fpr, tpr, auc_s = get_roc_auc(svc, X_test_bow, y_test_bow)

plt.plot(fpr, tpr, color='blue',
         lw=lw, 
         label=('SVC + BoW (area under the curve:{0:0.2f})'.format(auc_s)))

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")

plt.show()<jupyter_output><empty_output><jupyter_text>Emotion<jupyter_code>data_twit = pd.read_csv('data/Sentiment Analysis Dataset.csv', encoding = "ISO-8859-1")
data_twit.TEXT = data_twit.SentimentText.str.replace('[^\w\s]','')

def lemmatize_text(text):
    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]

data_twit['TEXT_L'] = data_twit.SentimentText.apply(lemmatize_text)
data_twit = data_twit[0:10000]
data_twit.head(10)<jupyter_output><empty_output><jupyter_text>Persistent homology<jupyter_code>from sklearn.svm import SVC
texts_ph = []

for element in data_twit['TEXT_L']:
    try:
        a = PersistentNonTrained(split_sent=element, min_count=1, workers=-1)
        dgms = a.persistent()
        texts_ph.append(flatten([[p.birth for p in dgms[1]], [p.death for p in dgms[1]]]))
    except:
        texts_ph.append([])
text_ph = pd.DataFrame(texts_ph)

X_twit = text_ph.fillna(0).values
y_twit = data_twit.Sentiment[0:10000]
svc_em = SVC(probability=True)
X_train_tw, X_test_tw, y_train_tw, y_test_tw = train_test_split(X_twit, y_twit, test_size=0.3)
svc_em.fit(X_train_tw, y_train_tw)
svc_em.score(X_test_tw, y_test_tw)<jupyter_output><empty_output><jupyter_text>Tf-idf<jupyter_code>word_vectorizer_2 = TfidfVectorizer(
    sublinear_tf=True,
    strip_accents='unicode',
    analyzer='word',
    token_pattern=r'\w{1,}',
    stop_words='english',
    ngram_range=(1, 2),
    max_features=10000)

text_tfidf = word_vectorizer_2.fit_transform(data_twit.SentimentText).toarray()

text_tfidf = pd.DataFrame(text_tfidf)
X_train_tw_tfidf, X_test_tw_tfidf, y_train_tw_tfidf, y_test_tw_tfidf = train_test_split(text_tfidf, y_twit, test_size=0.3)
lrc_em_tfidf = LogisticRegression()
lrc_em_tfidf.fit(X_train_tw_tfidf, y_train_tw_tfidf)
plt.figure(figsize=(10, 7))

lw = 2

fpr, tpr, auc_s = get_roc_auc(svc_em, X_test_tw, y_test_tw)

plt.plot(fpr, tpr, color='darkorange',
         lw=lw, 
         label=('Logitstic regression + w2v + PH (area under the curve:{0:0.2f})'.format(auc_s)))

fpr, tpr, auc_s = get_roc_auc(lrc_em_tfidf, X_test_tw_tfidf, y_test_tw_tfidf)

plt.plot(fpr, tpr, color='blue',
         lw=lw, 
         label=('Logitstic regression + tf-idf (area under the curve:{0:0.2f})'.format(auc_s)))

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")

plt.show()<jupyter_output><empty_output><jupyter_text>BoW<jupyter_code>bigram_vectorizer = CountVectorizer(ngram_range=(1, 2), token_pattern=r'\b\w+\b', min_df=1)

text_bow_tw = bigram_vectorizer.fit_transform(data_twit.SentimentText).toarray()

text_bow_tw_df = pd.DataFrame(text_bow_tw)
X_train_tw_bow, X_test_tw_bow, y_train_tw_bow, y_test_tw_bow = train_test_split(text_bow_tw_df , y_twit, test_size=0.3)
lrc_em_bow = LogisticRegression()
lrc_em_bow.fit(X_train_tw_bow, y_train_tw_bow)
plt.figure(figsize=(10, 7))

lw = 2

fpr, tpr, auc_s = get_roc_auc(svc_em, X_test_tw, y_test_tw)

plt.plot(fpr, tpr, color='darkorange',
         lw=lw, 
         label=('SVC + w2v + PH (area under the curve:{0:0.2f})'.format(auc_s)))

fpr, tpr, auc_s = get_roc_auc(lrc_em_tfidf, X_test_tw_tfidf, y_test_tw_tfidf)

plt.plot(fpr, tpr, color='blue',
         lw=lw, 
         label=('Logistic regression + tf-idf (area under the curve:{0:0.2f})'.format(auc_s)))

fpr, tpr, auc_s = get_roc_auc(lrc_em_bow, X_test_tw_bow, y_test_tw_bow)

plt.plot(fpr, tpr, color='green',
         lw=lw, 
         label=('Logitstic regression + BoW (area under the curve:{0:0.2f})'.format(auc_s)))

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")

plt.show()<jupyter_output><empty_output><jupyter_text>Word2vec vs. PH + word2vec  Non-trained<jupyter_code>texts_l = texts.TEXT_L

texts_w2v = []

for element in texts_l:
    try:
        a = PersistentNonTrained(split_sent=element, min_count=1, workers=-1)
        texts_w2v.append(flatten(a._get_vectors()))
    except TypeError:
        texts_w2v.append([])
        

text_w2v = pd.DataFrame(texts_w2v)
text_w2v.head(10)
text_w2v['ID'] = texts['ID']

X = pd.merge(train, text_w2v.fillna(0), on='ID', how='left').drop(['ID', 'CATEGORY'], 1).values
X.shape
X_train_w, X_test_w, y_train_w, y_test_w = train_test_split(X, pd.factorize(train['CATEGORY'])[0], test_size=0.3)
logist_w = LogisticRegression()
logist_w.fit(X_train_w, y_train_w)
logist_w.score(X_test_w, y_test_w)
plt.figure(figsize=(10, 7))

lw = 2

fpr, tpr, auc_s = get_roc_auc(logist, X_test, y_test)

plt.plot(fpr, tpr, color='darkorange',
         lw=lw, 
         label=('Logitstic regression + w2v + PH (area under the curve:{0:0.2f})'.format(auc_s)))


fpr, tpr, auc_s = get_roc_auc(logist_w, X_test_w, y_test_w)

plt.plot(fpr, tpr, color='blue',
         lw=lw, 
         label=('Logitstic regression + w2v (area under the curve:{0:0.2f})'.format(auc_s)))

plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc="lower right")

plt.show()
logist.score(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Pre-trained<jupyter_code>model = KeyedVectors.load_word2vec_format('persistent/GoogleNews-vectors'
                                              '-negative300.bin', binary=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15621868388820845, 'fraction_numerical': 0.021601158000222692, 'mean_word_length': 5.347703180212014, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '57272', 'license': 'MIT', 'n_tokens_mistral': 4061, 'n_tokens_neox': 3903, 'n_words': 583, 'path': 'Untitled2.ipynb', 'repo_name': 'alexeykosh/persistent_JA', 'size': '8981'}