<jupyter_start><jupyter_text>Prapare data to rnn<jupyter_code>vectorizer = KeyedVectors.load_word2vec_format(w2v_path, binary=False)

labels = []
for i, row in dataset_hh.iterrows():
    specs = row["profarea_names"].lower()
    specs = specs.split("', ") 
    for spec in specs: 
        spec = re.sub('[\[\'\]]', '', spec)
        labels.append(spec.strip())

labels = list(set(labels))
print("Labels number:", len(labels))
print("Labels:", labels)
def vectorize_text(text, vectorizer, max_len):
    """
    :param str text: normalized text
    :param Word2Vec vectorizer:
    :param int max_len:
    :return np.array text_vect: of shape like (1, max_len, 100)
    """
    text_vect = []
    words = text.split(" ")
    for word in words:
        try:
            word_vect = vectorizer.word_vec(word)
            text_vect.append(word_vect)
        except KeyError:
            None
      
    np.array(text_vect)
    text_vect = np.reshape(text_vect, (1, -1, 100))
    text_vect = sequence.pad_sequences(text_vect, maxlen=max_len, dtype='float')
            
    return text_vect
    

n_classes = len(labels)
n_examples = len(dataset_hh)
X = []
Y = np.zeros((n_examples, n_classes))
for i, row in dataset_hh.iterrows():
    X.append([])
    words = row["text_lemmas_tags"].split(" ")
    for word in words:
        try:
            word_vec = vectorizer.word_vec(word)
            X[i].append(word_vec)
        except KeyError:
            None
    
    specs = row["profarea_names"].lower() 
    specs = specs.split("', ") 
    for spec in specs: 
        spec = re.sub('[\[\'\]]', '', spec)
        pos = labels.index(spec.strip())
        Y[i][pos] = 1
        
assert(len(X) == len(Y))

max_len = max([len(i) for i in X])
print("Max len =", max_len)
max_len = 300
print("Max len edited=", max_len)
X = sequence.pad_sequences(X, maxlen=max_len, dtype='float')
X = np.array(X)

print(X.shape)
print(Y.shape)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=100)
print(X_train.shape)
print(X_test.shape)
print(Y_train.shape)
print(Y_test.shape)
print(Y_train.shape)
print(Y_test.shape)
del X
del Y
def get_classes(y_one_hot, labels, threshold=0.5):
    """
    :param np.array y_one_hot: shape like (1, n_classes)
    :param list of str labels:
    :return list of str classes: 
    """
    classes = []
    y_rounded = np.zeros(y_one_hot.shape)
    y_rounded[y_one_hot > threshold] = 1
    for i in range(len(labels)):
        if y_rounded[0][i] == 1:
            classes.append(labels[i])
            
    return classes
        
counts = []
for i in range(len(labels)):
    print(' Species {} appears {} times'.format(labels[i], np.sum(Y_train[:,i]) + np.sum(Y_test[:,i])))
    counts.append(np.sum(Y_train[:,i]) + np.sum(Y_test[:,i]))
print('###')
all_y = np.concatenate([Y_train, Y_test])
all_sum_y = np.sum(all_y, axis=1)
print('As a mean {} species appear on each recording (standard deviation {})'.format(np.mean(all_sum_y), np.std(all_sum_y)))
counts
counts[15]
sns.countplot(Y_train[:, 15])
plt.plot(counts)
Y1 = Y_train[:, 15]
Y1.shape
indexes = []
labels_new = []
counts_new = []

for i in range(len(labels)):
    if counts[i] > 50:
        labels_new.append(labels[i])
        counts_new.append(counts[i])
        indexes.append(i)
        
Y_train = Y_train[:, indexes]
print(len(labels_new))
print(labels_new)
print()
print(counts_new)
classes_weights = []
max_count = max(counts_new)
for i in range(len(counts_new)):
    weight = max_count / counts_new[i] 
    classes_weights.append(weight)

classes_weights = [classes_weights]  
print(classes_weights) 
print(len(classes_weights))<jupyter_output>[[22.822580645161292, 10.559701492537313, 1.0, 6.072961373390558, 7.075, 14.894736842105264, 14.894736842105264, 18.141025641025642, 5.0, 11.141732283464567, 2.7263969171483624, 1.5149892933618843, 4.879310344827586, 4.477848101265823, 24.82456140350877, 2.5634057971014492, 6.706161137440758, 4.66996699669967, 24.396551724137932, 5.1454545454545455, 9.188311688311689]]
1<jupyter_text>Training<jupyter_code>sample_weight = np.ones(Y1.shape)
sample_weight[Y1 == 1] = 5
print(len(sample_weight[sample_weight==5]))
Y_train.shape
len(labels_new)
model = Sequential()

model.add(Conv1D(filters=32, kernel_size=3, padding= 'same',
                 activation="relu", input_shape=(max_len, 300)))
model.add(Conv1D(filters=32, kernel_size=3, padding= 'same',
                 activation="relu"))
model.add(MaxPooling1D(pool_size=2))

#model.add(LSTM(10))
model.add(Bidirectional(layer=LSTM(50, dropout=0.3, recurrent_dropout=0.3),
                        input_shape=(max_len, 300)))
model.add(Dense(50, activation="relu", kernel_regularizer=l2(0.1)))
model.add(Dropout(0.5))
model.add(Dense(len(labels_new), activation="sigmoid", kernel_regularizer=l2(0.1)))

model.compile(loss="binary_crossentropy",
              optimizer="adam",
              metrics=["accuracy"],
              loss_weights=classes_weights)

model.summary()
%%time
model.fit(X_train, Y_train,
          batch_size=64, 
          epochs=3,
          validation_split=0.15,
          verbose=1)

np.round(model.predict(X_train[0].reshape((1, -1, 300))))
Y_train[0]
predict = model.predict(X_train)
np.round(predict[:10])
np.sum(np.round(predict))
%%time
scores = model.evaluate(X_test, Y_test, batch_size=64)
print("Test accuracy: %.2f%%" % (scores[1] * 100))

classif_list = []
train_score_list, test_score_list = [], []

# Training
for i in range(len(labels)):
    y_train_for_label = Y_train[:,i]
    new_classifier = GaussianNB()
    new_classifier.fit(X_train, y_train_for_label)
    classif_list.append(new_classifier)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code># Test & display results
for i in range(len(birds_list)):
    classif = classif_list[i]
    train_score = float('{0:.3f}'.format(classif.score(X_train, y_train[:,i])))
    test_score = float('{0:.3f}'.format(classif.score(X_test, y_test[:,i])))
    train_score_list.append(train_score)
    test_score_list.append(test_score)
    print('Detecting {} with {}% accuracy (training {}%)'.format(birds_list[i], 100*test_score, 100*train_score))

predict_train = np.zeros_like(y_train)
predict_test = np.zeros_like(y_test)
for i in range(len(birds_list)):
    classif = classif_list[i]
    predict_train[:,i] = classif.predict(X_train)
    predict_test[:,i] = classif.predict(X_test)
acc_train = 1 - np.sum(np.abs(predict_train - y_train))/(y_train.shape[0]*y_train.shape[1])
acc_test = 1 - np.sum(np.abs(predict_test - y_test))/(y_test.shape[0]*y_test.shape[1])
print('###')
print('Global accuracy: testing {}, training {}'.format(acc_test, acc_train))

well_labeled = 0
for i in range(len(y_train)):
    if np.sum(np.abs(y_train[i,:] - predict_train[i,:])) == 0:
        well_labeled +=1
print('Overall {} out of the {} training samples were well labeled'.format(well_labeled,len(y_train)))

well_labeled = 0
for i in range(len(y_test)):
    if np.sum(np.abs(y_test[i,:] - predict_test[i,:])) == 0:
        well_labeled +=1
print('Overall {} out of the {} testing samples were well labeled'.format(well_labeled,len(y_test)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1568873987147248, 'fraction_numerical': 0.05993294216261526, 'mean_word_length': 4.1245526127415895, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '135926', 'license': 'MIT', 'n_tokens_mistral': 2975, 'n_tokens_neox': 2705, 'n_words': 475, 'path': 'data/drafts/classification2_w2v_rnn+cnn.ipynb', 'repo_name': 'Vivikar/jooble_se', 'size': '7158'}