<jupyter_start><jupyter_text>Siamese Recurrent Neural NetworkThe general architecture of the model is based on [this tutorial](https://medium.com/mlreview/implementing-malstm-on-kaggles-quora-question-pairs-competition-8b31b0b16a07).The data we'll be using is a sample of jobs corresponding to the same group according to the [Standard Occupational Classification](https://www.bls.gov/soc/) . The data is in a tsv file and can be downloaded [here](https://www.onetcenter.org/dl_files/database/db_20_1_text/Sample%20of%20Reported%20Titles.txt).https://sorenbouma.github.io/blog/oneshot/https://deepmind.com/research/publications/one-shot-learning-memory-augmented-neural-networks/https://medium.com/mlreview/implementing-malstm-on-kaggles-quora-question-pairs-competition-8b31b0b16a07<jupyter_code>from io import StringIO
import requests
import pandas as pd

file_url = 'https://www.onetcenter.org/dl_files/database/db_20_1_text/Sample%20of%20Reported%20Titles.txt'
csv = StringIO(requests.get(file_url).text)
df = pd.read_csv(csv, sep='\t').head()
df.head()<jupyter_output><empty_output><jupyter_text>Build datasetLet's create positive samples with pairs of job titles corresponding to the same SOC, and negative examples with pairs of job titles sampled from different SOC codes.<jupyter_code>import itertools
jobs_left = []
jobs_right = []
target = []

soc_codes = df['O*NET-SOC Code'].unique()
for code in soc_codes:
    similar_jobs = df[df['O*NET-SOC Code'] == code]['Reported Job Title']
    positive_pairs = list(itertools.combinations(similar_jobs, 2))
    jobs_left.extend([p[0] for p in positive_pairs])
    jobs_right.extend([p[1] for p in positive_pairs])
    target.extend([1.]*len(positive_pairs))
    
    other_jobs = df[df['O*NET-SOC Code'] != code]['Reported Job Title']
    for i in range(len(positive_pairs)):
        jobs_left.append(np.random.choice(similar_jobs))
        jobs_right.append(np.random.choice(other_jobs))
        target.append(0.)

dataset = pd.DataFrame({
        'job_left': jobs_left,
        'job_right': jobs_right,
        'target': target
    }).sample(frac=1)  # Shuffle dataset

dataset.sample(5)
from sklearn.model_selection import train_test_split

df_train, df_val = train_test_split(dataset)<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>import re
from sklearn.pipeline import make_pipeline, FeatureUnion
from sklearn.preprocessing import FunctionTransformer
from zeugma import TextsToSequences, Padder, ItemSelector, EmbeddingTransformer

maxlen = 7
vocab_size = 10000

def preprocess_job_titles(job_titles):
    """ Return a list of clean job titles """
    def preprocess_job_title(raw_job_title):
        """ Clean a single job title"""
        job_title = re.sub(r'\(.*\)', '', raw_job_title)
        return job_title.lower().strip()
    return [preprocess_job_title(jt) for jt in job_titles]
    
pipeline = make_pipeline(
    FunctionTransformer(preprocess_job_titles, validate=False),
    TextsToSequences(num_words=vocab_size), 
    Padder(max_length=maxlen),
)

pipeline.fit(list(df_train['job_left'])+list(df_train['job_right']))
X_left_train = pipeline.transform(df_train['job_left'])
X_right_train = pipeline.transform(df_train['job_right'])
seq_train = [X_left_train, X_right_train]

y_train = df_train['target'].values
import numpy as np
from keras.models import Model
from keras.layers import concatenate, Flatten
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, GlobalMaxPooling1D
from keras.layers import Conv1D, MaxPooling1D, Embedding, Dropout

glove = EmbeddingTransformer('glove')
EMBEDDING_DIM = glove.model.get('the').shape[0]

word_index = pipeline.get_params()['textstosequences'].word_index

def create_embedding_matrix(vocab_size, word_index, embedding_dim=EMBEDDING_DIM):
    """ Prepare embedding matrix """
    num_words = min(vocab_size, len(word_index))
    embedding_matrix = np.zeros((num_words+1, embedding_dim))
    for word, i in word_index.items():
        if i >= vocab_size:
            continue
        embedding_vector = glove.model.get(word)
        if embedding_vector is not None:
            # words not found in embedding index will be all-zeros.
            embedding_matrix[i] = embedding_vector
    return embedding_matrix

embedding_matrix = create_embedding_matrix(vocab_size, word_index)
from keras.layers import LSTM, Bidirectional
from keras import Model, Sequential
from keras.layers import Input, Dense, Dropout, Lambda, Subtract, Merge
from keras import backend as K

def exponent_neg_manhattan_distance(arms_difference):
    return K.exp(-K.sum(K.abs(arms_difference), axis=1, keepdims=True))

def siamese_lstm(maxlen=maxlen):
    """ Define, compile and return a siamese LSTM model """
    input_shape = (maxlen,)
    left_input = Input(input_shape, name='left_input')
    right_input = Input(input_shape, name='right_input')

    # load pre-trained word embeddings into an Embedding layer
    # note that we set trainable = False so as to keep the embeddings fixed
    embedding_layer = Embedding(len(embedding_matrix),
                                EMBEDDING_DIM,
                                weights=[embedding_matrix],
                                input_length=maxlen,
                                trainable=False,
                                name='embeddings')

    seq = Sequential()
    seq.add(embedding_layer)
    seq.add(Bidirectional(LSTM(64, dropout=0.2, recurrent_dropout=0.2,
                               return_sequences=True)))
    seq.add(Bidirectional(LSTM(64, dropout=0.2, recurrent_dropout=0.2,)))
    
    left_output = seq(left_input)
    right_output = seq(right_input)

    subtracted = Subtract(name='subtract')([left_output, right_output])
    malstm_distance = Lambda(exponent_neg_manhattan_distance, 
                             name='masltsm_distance')(subtracted)

    siamese_net = Model(inputs=[left_input, right_input], outputs=malstm_distance)

    siamese_net.compile(loss="binary_crossentropy", optimizer='adam',
                        metrics=['accuracy'])
    return siamese_net

siamese_lstm = siamese_lstm()

siamese_lstm.summary()
siamese_lstm.fit(seq_train, y_train, validation_split=0.2, epochs=10,)
X_left_val = pipeline.transform(df_val['job_left'])
X_right_val = pipeline.transform(df_val['job_right'])
seq_val = [X_left_val, X_right_val]

y_val = df_val['target'].values
y_prob = siamese_lstm.predict(seq_val)
y_pred = np.round(y_prob)
from sklearn.metrics import accuracy_score, roc_auc_score
roc_auc_score(y_val, y_prob)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10569105691056911, 'fraction_numerical': 0.01038843721770551, 'mean_word_length': 4.418433931484502, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 7, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '567215', 'license': 'MIT', 'n_tokens_mistral': 2240, 'n_tokens_neox': 2168, 'n_words': 435, 'path': 'examples/siamese_rnn.ipynb', 'repo_name': 'Y-Sree-Chaitanya/zeugma', 'size': '6642'}