<jupyter_start><jupyter_text>End of definitions------------------Start data preparation<jupyter_code>importdf=pd.read_csv('../data/trainingssets/all_emoji_tweets_03_12_18_7_labels_excluded.csv', sep =';', usecols=['tweet_full_text', 'target'])
importdf.dropna(inplace=True)
importdf.reset_index(inplace=True, drop=True)
all_targets = importdf['target'].astype(str).values.tolist()
y=[]
for i in range(len(all_targets)):
    #Only use first emoji per tweet for now
    y.append(all_targets[i].split(',')[0])

# for filtering in conversion to binary classification later on
dfy=pd.DataFrame(y)
dfx=pd.DataFrame(importdf['tweet_full_text'])
dfx.columns = range(dfx.shape[1])
# convert to binary classification
binary_labels=['♥️', '😂']   # two Labels chosen for binary classification

dfy=dfy[dfy.isin(binary_labels)]
dfy.dropna(inplace=True)
dfx=dfx[dfy.isin(binary_labels)]
dfx.dropna(inplace=True)

df=dfx.copy()
df.rename(inplace=True, columns={0: "tweet"})
df['target'] = dfy

# balance classes to 50:50 by dropping appropriate (randomized) fraction of majority class
majority_class='♥️'
class_freq=df['target'].value_counts()
df = df.drop(df[df['target'] == majority_class].sample(frac=(1-class_freq[1]/class_freq[0]), random_state=123).index)

# prepare data for following steps
our_tweets=df['tweet'].astype(str).values.tolist()
y=df['target']
y=np.array(y)
corpus=[]
for i in range(len(our_tweets)):
    corpus.append(clean_tweets(our_tweets[i]))
corpus<jupyter_output><empty_output><jupyter_text>End data preparation----------------------------Start Word2Vec preparation<jupyter_code>word2vec_model_path = "../data/word2vec/tweets_until_29_11_18_model3.model"
word2vec_model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_model_path, binary=True)
#word2vec_model['zwei'] #klass aktion zwei gelbew hang bruck revolution ahu
#w2vKeyedVectors to dict. Necessary in vectorizer, because w2vKeyedVectors has no attribute 'itervalues'
w2v = {w: vec for w, vec in zip(word2vec_model.index2word, word2vec_model.vectors)} 

corpus_sentences = CorpusSentenceTokenList(corpus)
phrases = gensim.models.Phrases(corpus_sentences, threshold=50, min_count=20)
corpus_phrases = [sentence for sentence in phrases[corpus_sentences]]<jupyter_output><empty_output><jupyter_text>End word2vec preparation--------------------------Start data visualisation<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(corpus, y, test_size=0.4, random_state=0)
pipe_transformer = Pipeline([
    ('vect', MeanEmbeddingVectorizer(w2v))
])
pipe_transformer.fit(X_train, y_train);
fig = plt.figure(figsize=(16, 16))          
plot_LSA(pipe_transformer.transform(X_train), y_train)
plt.savefig('../figures/tweets_from_03_12_18/binaryclass/word2vec_meanembeddingvectorizer_lsa')
plt.show()
X_train, X_test, y_train, y_test = train_test_split(corpus, y, test_size=0.4, random_state=0)
pipe_transformer = Pipeline([
    ('vect', TfidfEmbeddingVectorizer(w2v))
])
pipe_transformer.fit(X_train, y_train);
fig = plt.figure(figsize=(16, 16))          
plot_LSA(pipe_transformer.transform(X_train), y_train)
plt.savefig('../figures/tweets_from_03_12_18/binaryclass/word2vec_tfidfembeddingvectorizer_lsa')
plt.show()<jupyter_output><empty_output><jupyter_text>For use after arbitrary GridSearchNeeds to be run twice to work? Probably some mistake herefig = plt.figure(figsize=(16, 16))          clf = gs_clf.best_estimator_.steps.pop(1)plot_LSA(gs_clf.best_estimator_.transform(X_train), y_train)gs_clf.best_estimator_.steps.append(clf)plt.show() End data visualisation--------------------------Start SGDClassifier<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(corpus_phrases, y, test_size=0.4, random_state=0)
pipe_clf = Pipeline([
    ('vect', TfidfEmbeddingVectorizer(w2v)),
    ('clf', SGDClassifier(random_state=0, 
                          max_iter=100, tol=None))
])

parameters = {
     'clf__loss': ['modified_huber','log'], #modified_huber
     'clf__penalty': ['elasticnet','l2'], #elasticnet
     'clf__alpha': [1e-5],
     'clf__epsilon': [0.01],
     'clf__learning_rate': ['invscaling','optimal'],
     'clf__eta0': [10]
     #'clf__eta0': [1e-4, 0.1],
}
    
gs_clf = GridSearchCV(pipe_clf, parameters, cv=3, iid=False, n_jobs=-1, verbose=1)
gs_clf.fit(X_train, y_train)
predicted = gs_clf.predict(X_test)
np.mean(predicted == y_test)<jupyter_output><empty_output><jupyter_text>Test run: Tweets from 29.11:run1:    * modified_huber    * elasticnet    * alpha=1e-5    * epsilon=0.01    * invscaling    * eta0=10    * TfidfEmbeddingVectorizer    * tweets_until_29_11_18_model1.model    Score 0.7065608698846522run2:    * modified_huber    * elasticnet    * alpha=1e-5    * epsilon=0.01    * invscaling    * eta0=10    * TfidfEmbeddingVectorizer    * german.model (clean_tweets modified)Score 0.6587319080155757run3:    * modified_huber    * elasticnet    * alpha=1e-5    * epsilon=0.01    * invscaling    * eta0=10    * TfidfEmbeddingVectorizer    * tweets_until_29_11_18_model3.modelScore 0.7092425244287709<jupyter_code>for param_name in sorted(parameters.keys()):
    print("%s: %r" % (param_name, gs_clf.best_params_[param_name]))
plt.figure()
cnf_matrix = confusion_matrix(y_test, predicted)
plot_confusion_matrix(cnf_matrix, classes=binary_labels, normalize=True,
                      title='Confusion matrix, with normalization')
plt.savefig('../figures/tweets_from_03_12_18/binaryclass/word2vec_tfidfembeddingvectorizer_sgdclassifier_confusion_matrix')
plt.show()<jupyter_output><empty_output><jupyter_text>End Advanced SGDClassifier----------------------------Start Advanced RandomForest<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(corpus, y, test_size=0.4, random_state=0)
pipe_clf = Pipeline([
    ('vect', TfidfEmbeddingVectorizer(w2v)),
    ('clf', RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=0))
])

parameters = {
     #'vect__ngram_range': [(1, 1), (1, 2)],
     #'vect__max_df': [0.9, 1.0],
     #'tfidf__use_idf': (True, False)''
     #'clf__criterion': ['gini', 'entropy'],
     #'clf__max_features': ['log2', 'auto', 0.5],
}
gs_clf = GridSearchCV(pipe_clf, parameters, cv=3, iid=False, n_jobs=-1)
gs_clf.fit(X_train, y_train)
predicted = gs_clf.predict(X_test)
np.mean(predicted == y_test)  
for param_name in sorted(parameters.keys()):
    print("%s: %r" % (param_name, gs_clf.best_params_[param_name]))
plt.figure()
cnf_matrix = confusion_matrix(y_test, predicted)
plot_confusion_matrix(cnf_matrix, classes=binary_labels, normalize=True,
                      title='Confusion matrix, with normalization')
plt.savefig('../figures/tweets_from_03_12_18/binaryclass/tfidftransformer_randomforestclassifier_confusion_matrix')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1497849621830046, 'fraction_numerical': 0.03499925849028622, 'mean_word_length': 5.637795275590551, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '161645', 'license': 'MIT', 'n_tokens_mistral': 2670, 'n_tokens_neox': 2528, 'n_words': 384, 'path': 'python/skripte_klassifikation/binary_classification_word2vec.ipynb', 'repo_name': 'TimSchmittmann/R-vs-Python', 'size': '6743'}