<jupyter_start><jupyter_text>Data Processing  Use pre-trained word embeddings based on patent data<jupyter_code>word2vec_model = KeyedVectors.load_word2vec_format('uspto_2m_abstract_word2vec.bin', binary=True)
#read text data and their labels
label_words_df = pd.DataFrame(pd.read_csv('uspto_2m_abstr_label_valid_label.tsv', sep='\t'))
label_words_df.head()
#set up balanced dataset
size = 500        # sample size
replace = True  # cannot choose False, since some classes do not have 100 samples
fn = lambda obj: obj.loc[np.random.choice(obj.index, size, replace),:]
sub_df = label_words_df.groupby('label', as_index=False).apply(fn)
y = sub_df['label']
len(sub_df)
sub_df.columns
#extract words
from tensorflow.python.keras.preprocessing.text import Tokenizer
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from sklearn.preprocessing import LabelEncoder

tokenizer = Tokenizer()
#tokenizer = Tokenizer(num_words=20)
tokenizer.fit_on_texts(sub_df['Abstract'].values)
sequences = tokenizer.texts_to_sequences(sub_df['Abstract'].values)


#max length for pad sequences
seq_max_len = max([len(s.split()) for s in sub_df['Abstract'].values])

word_index = tokenizer.word_index
print ('Number of unique tokens: ', len(word_index))

x_pad = pad_sequences(sequences, maxlen=seq_max_len)
encoder = LabelEncoder()
y_set = encoder.fit_transform(y)<jupyter_output>Number of unique tokens:  63929<jupyter_text>Prepare the embedding layer<jupyter_code>embedding_dim = word2vec_model.vector_size
embedding_matrix = np.zeros((len(word_index) + 1, embedding_dim))
for word, i in word_index.items():
    if word in word2vec_model:
        embedding_matrix[i] = word2vec_model[word]
    #print (word, i)<jupyter_output><empty_output><jupyter_text>Train LSTM, GRU, CNN with the pre-trained embedding layer CNN is faster than LSTM, GRU.Future work:1. Try seqCNN, etc., other CNN variations.2. Try large dataset using CNN, compared to Naive Bayes, SVM.<jupyter_code>from tensorflow.python.keras.layers import Embedding, Dense, LSTM, GRU
from tensorflow.python.keras.layers import Conv1D, MaxPooling1D, Flatten
from tensorflow.python.keras.layers import Dropout, SeparableConv1D, GlobalAveragePooling1D
from tensorflow.python.keras import models, callbacks
from tensorflow.python.keras.optimizers import Adam
numOfclasses = len(set(y_set))
last_layer_activation = 'softmax'
hidden_layer_activation = 'relu'
dropout_rate = 0.2
loss = 'sparse_categorical_crossentropy'
learning_rate = 0.001
epochs = 2
n_layers = 1
units = 64
batch_size = 128
#LSTM
#for output layer
model = models.Sequential()
#use pre-trained embedding matrix
model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], 
                    input_length=seq_max_len, trainable=False))

model.add(LSTM(units=units, dropout=dropout_rate, recurrent_dropout=dropout_rate))
model.add(Dense(units=numOfclasses, activation=last_layer_activation))
model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])
history = model.fit(x_pad, y_set, epochs=epochs, batch_size=batch_size, validation_split=0.1, 
                    verbose=1)
#GRU
epochs = 1
model = models.Sequential()
#use pre-trained embedding matrix
model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], 
                    input_length=seq_max_len, trainable=False))

model.add(GRU(units=units, dropout=dropout_rate, recurrent_dropout=dropout_rate))
model.add(Dense(units=numOfclasses, activation=last_layer_activation))
model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])
history = model.fit(x_pad, y_set, epochs=epochs, batch_size=batch_size, validation_split=0.1, 
                    verbose=1)
#CNN
model = models.Sequential()
model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], 
                    input_length=seq_max_len, trainable=False))
model.add(Conv1D(250, 5, activation=hidden_layer_activation))
model.add(MaxPooling1D(pool_size=4))
model.add(Flatten())
model.add(Dense(units=numOfclasses, activation=last_layer_activation))
model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])
history = model.fit(x_pad, y_set, epochs=epochs, batch_size=batch_size, validation_split=0.1, verbose=1)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x_pad, y_set, test_size = 0.2, random_state = 0)
#adding more layers get worse results
epochs = 20
model = models.Sequential()
model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], 
                    input_length=seq_max_len, trainable=False))
model.add(Conv1D(128, 3, activation=hidden_layer_activation))
model.add(MaxPooling1D(pool_size=2))  #vary 2,3,4,5
model.add(Flatten())
model.add(Dense(units=numOfclasses, activation=last_layer_activation))
model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test), verbose=1)<jupyter_output>Train on 252000 samples, validate on 63000 samples
Epoch 1/20
252000/252000 [==============================] - 510s 2ms/sample - loss: 2.9933 - accuracy: 0.4491 - val_loss: 2.0919 - val_accuracy: 0.5856
Epoch 2/20
252000/252000 [==============================] - 551s 2ms/sample - loss: 1.2979 - accuracy: 0.7330 - val_loss: 1.9458 - val_accuracy: 0.6271
Epoch 3/20
252000/252000 [==============================] - 545s 2ms/sample - loss: 0.9508 - accuracy: 0.8056 - val_loss: 1.9214 - val_accuracy: 0.6333
Epoch 4/20
252000/252000 [==============================] - 549s 2ms/sample - loss: 0.8053 - accuracy: 0.8299 - val_loss: 1.9267 - val_accuracy: 0.6381
Epoch 5/20
252000/252000 [==============================] - 545s 2ms/sample - loss: 0.7241 - accuracy: 0.8378 - val_loss: 1.9210 - val_accuracy: 0.6405
Epoch 6/20
252000/252000 [==============================] - 547s 2ms/sample - loss: 0.6618 - accuracy: 0.8418 - val_loss: 1.9227 - val_accuracy: 0.6401
Epoch 7/20
252000/252000 [===========[...]<jupyter_text>Train CNN and allows to learn the embedding layer<jupyter_code>#CNN without weights at the embedding layer
model = models.Sequential()
model.add(Embedding(len(word_index) + 1, embedding_dim, input_length=seq_max_len))
model.add(Conv1D(128, 3, activation=hidden_layer_activation))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(units=numOfclasses, activation=last_layer_activation))
model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test), verbose=1)<jupyter_output>Train on 252000 samples, validate on 63000 samples
Epoch 1/20
252000/252000 [==============================] - 718s 3ms/sample - loss: 3.4053 - accuracy: 0.4093 - val_loss: 2.2308 - val_accuracy: 0.5702
Epoch 2/20
252000/252000 [==============================] - 718s 3ms/sample - loss: 1.3872 - accuracy: 0.7234 - val_loss: 2.0174 - val_accuracy: 0.6224
Epoch 3/20
252000/252000 [==============================] - 719s 3ms/sample - loss: 0.9614 - accuracy: 0.8102 - val_loss: 2.0157 - val_accuracy: 0.6290
Epoch 4/20
252000/252000 [==============================] - 728s 3ms/sample - loss: 0.8019 - accuracy: 0.8343 - val_loss: 1.9627 - val_accuracy: 0.6305
Epoch 5/20
252000/252000 [==============================] - 727s 3ms/sample - loss: 0.7086 - accuracy: 0.8416 - val_loss: 1.9531 - val_accuracy: 0.6352
Epoch 6/20
252000/252000 [==============================] - 728s 3ms/sample - loss: 0.6447 - accuracy: 0.8447 - val_loss: 1.9782 - val_accuracy: 0.6325
Epoch 7/20
252000/252000 [===========[...]
{'fraction_non_alphanumeric': 0.16453549060542796, 'fraction_numerical': 0.07933194154488518, 'mean_word_length': 5.974522292993631, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '610583', 'license': 'MIT', 'n_tokens_mistral': 3087, 'n_tokens_neox': 2612, 'n_words': 560, 'path': 'textClassification_sequenceNNs.ipynb', 'repo_name': 'bowenyan/patent-text-classification', 'size': '7664'}