Spaces:
Build error
Build error
from keras.preprocessing.sequence import pad_sequences | |
from keras.preprocessing.text import Tokenizer | |
from gensim.models import Word2Vec | |
import numpy as np | |
import gc | |
def train_word2vec(documents, embedding_dim): | |
""" | |
train word2vector over training documents | |
Args: | |
documents (list): list of document | |
embedding_dim (int): output wordvector size | |
Returns: | |
word_vectors(dict): dict containing words and their respective vectors | |
""" | |
model = Word2Vec(documents, min_count=1, size=embedding_dim) | |
word_vectors = model.wv | |
del model | |
return word_vectors | |
def create_embedding_matrix(tokenizer, word_vectors, embedding_dim): | |
""" | |
Create embedding matrix containing word indexes and respective vectors from word vectors | |
Args: | |
tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object containing word indexes | |
word_vectors (dict): dict containing word and their respective vectors | |
embedding_dim (int): dimension of word vector | |
Returns: | |
""" | |
nb_words = len(tokenizer.word_index) + 1 | |
word_index = tokenizer.word_index | |
embedding_matrix = np.zeros((nb_words, embedding_dim)) | |
print("Embedding matrix shape: %s" % str(embedding_matrix.shape)) | |
for word, i in word_index.items(): | |
try: | |
embedding_vector = word_vectors[word] | |
if embedding_vector is not None: | |
embedding_matrix[i] = embedding_vector | |
except KeyError: | |
print("vector not found for word - %s" % word) | |
print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0)) | |
return embedding_matrix | |
def word_embed_meta_data(documents, embedding_dim): | |
""" | |
Load tokenizer object for given vocabs list | |
Args: | |
documents (list): list of document | |
embedding_dim (int): embedding dimension | |
Returns: | |
tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object | |
embedding_matrix (dict): dict with word_index and vector mapping | |
""" | |
documents = [str(x).lower().split() for x in documents] | |
tokenizer = Tokenizer() | |
tokenizer.fit_on_texts(documents) | |
word_vector = train_word2vec(documents, embedding_dim) | |
embedding_matrix = create_embedding_matrix(tokenizer, word_vector, embedding_dim) | |
del word_vector | |
gc.collect() | |
return tokenizer, embedding_matrix | |
def create_train_dev_set(tokenizer, sentences_pair, is_similar, max_sequence_length, validation_split_ratio): | |
""" | |
Create training and validation dataset | |
Args: | |
tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object | |
sentences_pair (list): list of tuple of sentences pairs | |
is_similar (list): list containing labels if respective sentences in sentence1 and sentence2 | |
are same or not (1 if same else 0) | |
max_sequence_length (int): max sequence length of sentences to apply padding | |
validation_split_ratio (float): contain ratio to split training data into validation data | |
Returns: | |
train_data_1 (list): list of input features for training set from sentences1 | |
train_data_2 (list): list of input features for training set from sentences2 | |
labels_train (np.array): array containing similarity score for training data | |
leaks_train(np.array): array of training leaks features | |
val_data_1 (list): list of input features for validation set from sentences1 | |
val_data_2 (list): list of input features for validation set from sentences1 | |
labels_val (np.array): array containing similarity score for validation data | |
leaks_val (np.array): array of validation leaks features | |
""" | |
sentences1 = [x[0].lower() for x in sentences_pair] | |
sentences2 = [x[1].lower() for x in sentences_pair] | |
train_sequences_1 = tokenizer.texts_to_sequences(sentences1) | |
train_sequences_2 = tokenizer.texts_to_sequences(sentences2) | |
leaks = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))] | |
for x1, x2 in zip(train_sequences_1, train_sequences_2)] | |
train_padded_data_1 = pad_sequences(train_sequences_1, maxlen=max_sequence_length) | |
train_padded_data_2 = pad_sequences(train_sequences_2, maxlen=max_sequence_length) | |
train_labels = np.array(is_similar) | |
leaks = np.array(leaks) | |
shuffle_indices = np.random.permutation(np.arange(len(train_labels))) | |
train_data_1_shuffled = train_padded_data_1[shuffle_indices] | |
train_data_2_shuffled = train_padded_data_2[shuffle_indices] | |
train_labels_shuffled = train_labels[shuffle_indices] | |
leaks_shuffled = leaks[shuffle_indices] | |
dev_idx = max(1, int(len(train_labels_shuffled) * validation_split_ratio)) | |
del train_padded_data_1 | |
del train_padded_data_2 | |
gc.collect() | |
train_data_1, val_data_1 = train_data_1_shuffled[:-dev_idx], train_data_1_shuffled[-dev_idx:] | |
train_data_2, val_data_2 = train_data_2_shuffled[:-dev_idx], train_data_2_shuffled[-dev_idx:] | |
labels_train, labels_val = train_labels_shuffled[:-dev_idx], train_labels_shuffled[-dev_idx:] | |
leaks_train, leaks_val = leaks_shuffled[:-dev_idx], leaks_shuffled[-dev_idx:] | |
return train_data_1, train_data_2, labels_train, leaks_train, val_data_1, val_data_2, labels_val, leaks_val | |
def create_test_data(tokenizer, test_sentences_pair, max_sequence_length): | |
""" | |
Create training and validation dataset | |
Args: | |
tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object | |
test_sentences_pair (list): list of tuple of sentences pairs | |
max_sequence_length (int): max sequence length of sentences to apply padding | |
Returns: | |
test_data_1 (list): list of input features for training set from sentences1 | |
test_data_2 (list): list of input features for training set from sentences2 | |
""" | |
test_sentences1 = [str(x[0]).lower() for x in test_sentences_pair] | |
test_sentences2 = [x[1].lower() for x in test_sentences_pair] | |
test_sequences_1 = tokenizer.texts_to_sequences(test_sentences1) | |
test_sequences_2 = tokenizer.texts_to_sequences(test_sentences2) | |
leaks_test = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))] | |
for x1, x2 in zip(test_sequences_1, test_sequences_2)] | |
leaks_test = np.array(leaks_test) | |
test_data_1 = pad_sequences(test_sequences_1, maxlen=max_sequence_length) | |
test_data_2 = pad_sequences(test_sequences_2, maxlen=max_sequence_length) | |
return test_data_1, test_data_2, leaks_test | |