<jupyter_start><jupyter_text><jupyter_code>import random
import numpy as np
import tensorflow as tf


def prepareData(inputs: np.ndarray, labels: np.ndarray):
      classesNumbers = 10
      digitalIdx = [np.where(labels == i)[0] for i in range(classesNumbers)]
      pairs = list()
      labels = list()
      n = min([len(digitalIdx[d]) for d in range(classesNumbers)]) - 1
      for d in range(classesNumbers):
        for i in range(n):
            z1, z2 = digitalIdx[d][i], digitalIdx[d][i + 1]
            pairs += [[inputs[z1], inputs[z2]]]
            inc = random.randrange(1, classesNumbers)
            dn = (d + inc) % classesNumbers
            z1, z2 = digitalIdx[d][i], digitalIdx[dn][i]
            pairs += [[inputs[z1], inputs[z2]]]
            labels += [1, 0]    
      return np.array(pairs), np.array(labels, dtype=np.float32)
def createTemplate():
      return tf.keras.models.Sequential([
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(128, activation='relu'),
        tf.keras.layers.Dropout(0.15),
        tf.keras.layers.Dense(128, activation='relu'),
        tf.keras.layers.Dropout(0.15),
        tf.keras.layers.Dense(64, activation='relu'),    
        ])
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
x_train = x_train.astype(np.float32)
x_test = x_test.astype(np.float32)
x_train /= 255
x_test /= 255
input_shape = x_train.shape[1:]
train_pairs, tr_labels = prepareData(x_train, y_train)
test_pairs, test_labels = prepareData(x_test, y_test)
base_network = createTemplate()
# Create first half of the siamese system
input_a = tf.keras.layers.Input(shape=input_shape)
# Note how we reuse the base_network in both halfs
enconder1 = base_network(input_a)
# Create the second half of the siamese system
input_b = tf.keras.layers.Input(shape=input_shape)
enconder2 = base_network(input_b)
distance = tf.keras.layers.Lambda(    
    lambda embeddings: tf.keras.backend.abs(embeddings[0] - embeddings[1])) \
    ([enconder1, enconder2])
measureOfSimilarity = tf.keras.layers.Dense(1, activation='sigmoid') (distance)
# Build the model
model = tf.keras.models.Model([input_a, input_b], measureOfSimilarity)
# Train
model.compile(loss='binary_crossentropy',optimizer=tf.keras.optimizers.Adam(),metrics=['accuracy'])

model.fit([train_pairs[:, 0], train_pairs[:, 1]], tr_labels,          
          batch_size=128,epochs=10,validation_data=([test_pairs[:, 0], test_pairs[:, 1]], test_labels))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14239871640593663, 'fraction_numerical': 0.023666265543521862, 'mean_word_length': 3.796153846153846, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '225908', 'license': 'MIT', 'n_tokens_mistral': 913, 'n_tokens_neox': 875, 'n_words': 173, 'path': 'siamese_system.ipynb', 'repo_name': 'biozid-0208/daily-random-programming-commit', 'size': '2493'}