<jupyter_start><jupyter_text>Dataset Details - For adaptation between MNIST and USPS, we follow the training protocol established in [21], __sampling 2000 images from MNIST and 1800 from USPS.__  Model Details - our adversarial discriminator consists of 3 fully connected layers: two layers with 500 hidden units followed by the final discriminator output - Each of the 500-unit layers uses a ReLU activation function<jupyter_code>def LeNetEncoder():
    model = tf.keras.Sequential()
    
    model.add(layers.Conv2D(20, (5, 5),
                        activation='relu',
                        input_shape=(28, 28, 3)))
    model.add(layers.MaxPooling2D((2, 2)))
    
    model.add(layers.Conv2D(50, (5, 5),
                        activation='relu',
                        input_shape=(28, 28, 3)))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Flatten())
    model.add(layers.Dropout(0.1))
    
    model.add(layers.Dense(500,
                           activation='relu'))
    
    return model
def LeNetClassifier():
    model = tf.keras.Sequential()
    model.add(layers.Dropout(0.1,
                             input_shape=(500,)))
    model.add(layers.Dense(10,
                           activation='softmax'))
    
    return model
def Critics():
    model = tf.keras.Sequential()
    model.add(layers.Dense(500,
                           input_shape=(500,),
                           activation='relu'))
    model.add(layers.Dense(500,
                           activation='relu'))
    model.add(layers.Dense(2,
                           activation='softmax'))
    
    return model
source_encoder = LeNetEncoder()
target_encoder = LeNetEncoder()
source_classifier = LeNetClassifier()
mnist_train_set = tf.data.Dataset.from_tensor_slices((mnist_X, mnist_y)).shuffle(buffer_size=mnist_X.shape[0]).batch(64)
mnist_test_set = tf.data.Dataset.from_tensor_slices((mnist_X_, mnist_y_)).shuffle(buffer_size=mnist_X_.shape[0]).batch(64)
usps_train_set = tf.data.Dataset.from_tensor_slices((usps_X, usps_y)).shuffle(buffer_size=usps_X.shape[0]).batch(64)
usps_test_set = tf.data.Dataset.from_tensor_slices((usps_X_, usps_y_)).shuffle(buffer_size=usps_X_.shape[0]).batch(64)
mnistm_train_set = tf.data.Dataset.from_tensor_slices((mnistm_X, mnist_y)).shuffle(buffer_size=mnist_X.shape[0]).batch(64)
mnistm_test_set = tf.data.Dataset.from_tensor_slices((mnistm_X_, mnist_y_)).shuffle(buffer_size=mnist_X_.shape[0]).batch(64)
opt = keras.optimizers.Adam()
# pre-train (1) source_encoder, (2) source_classifier
@tf.function
def pretrain_step(X, y):
    with tf.GradientTape() as tape:
        embedding = source_encoder(X)
        pred = source_classifier(embedding)
        loss_value = loss(y, pred)
        
    tvars = source_encoder.trainable_variables + source_classifier.trainable_variables
    grads = tape.gradient(loss_value, tvars)
    opt.apply_gradients(zip(grads, tvars))
    
    return pred, loss_value
loss = keras.losses.SparseCategoricalCrossentropy()
def model_eval(mydata, encoder, verbose=True):
    avg = keras.metrics.Mean()
    accuracy = keras.metrics.SparseCategoricalAccuracy()
    for X, y in mydata:
        # inference
        ebd = encoder(X)
        prob = source_classifier(ebd)
        lv = loss(y, prob)

        # results
        avg(lv)
        accuracy(y, prob)
    
    loss_v = avg.result().numpy()
    accuracy_v = accuracy.result().numpy()
    
    if verbose:
        print("Loss: {:.4}".format(loss_v), end=" | ")
        print("Accuracy: {:.2%}".format(accuracy_v))
    
    return loss_v, accuracy_v
def pre_train(mydata, epochs, lr):
    learning_rate = lr
    
    for epoch in range(epochs):
        for X, y in mydata:
            pred, lv = pretrain_step(X, y)
            pred_idx = tf.argmax(pred, axis=1)
            
            learning_rate = learning_rate / (1 + epoch/10)
            opt = tf.keras.optimizers.Adam(learning_rate=0.01)

        l, a = model_eval(mnist_test_set, source_encoder, False)
        print("Epoch:{}, Loss:{:.4}, Accuracy:{:.4}"
              .format(epoch+1,l,a))
pre_train(mnist_train_set, 20, 0.01)
# Check Source Only Performance
print("MNIST TEST: ", end=' ')
model_eval(mnist_test_set, source_encoder)
print("USPS TEST: ", end=' ')
model_eval(usps_test_set, source_encoder)
print("MNISTM TEST: ", end=' ')
model_eval(mnistm_test_set, source_encoder)
source_encoder.save('mnist_train.h5')
target_encoder = keras.models.load_model('mnist_train.h5', compile=False)
# source_encoder = keras.models.load_model('mnist_train.h5')
# Check Model Load
model_eval(mnist_test_set, source_encoder)
model_eval(mnist_test_set, target_encoder)
critic = Critics()
cce = tf.keras.losses.SparseCategoricalCrossentropy()
@tf.function
def train_step(img_src, img_tgt):
    # discriminator
    with tf.GradientTape() as disc_tape:
        source_ebd = source_encoder(img_src, training=False)
        target_ebd = target_encoder(img_tgt, training=True)
        concat_ebd = tf.concat([source_ebd, target_ebd], 0)

        concat_pred = critic(concat_ebd, training=True)
        
        source_label = tf.ones_like(source_ebd[:, 0])
        target_label = tf.zeros_like(target_ebd[:, 0])
        concat_label = tf.concat([source_label, target_label], 0)
        
        disc_loss = cce(concat_label, concat_pred)

    disc_grads = disc_tape.gradient(disc_loss, critic.trainable_variables)
    disc_opt.apply_gradients(zip(disc_grads, critic.trainable_variables))
    
    with tf.GradientTape() as tgt_tape:
        target_ebd = target_encoder(img_tgt, training=True)
        target_pred = critic(target_ebd, training=False)
        target_label = tf.ones_like(target_pred[:, 0])
        
        target_loss = cce(target_label, target_pred)
        
    target_grads = tgt_tape.gradient(target_loss, target_encoder.trainable_variables)
    tgt_opt.apply_gradients(zip(target_grads, target_encoder.trainable_variables))
    
    return disc_loss, target_loss
checkpoint_dir = './training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")

dll = []
tll = []
tgt_opt.learning_rate.get_config()
lr = 1e-5
tgt_opt = keras.optimizers.Adam(lr)
disc_opt = keras.optimizers.Adam(lr)

for epoch in range(100):
    for (sx, sy), (tx, ty) in zip(mnist_train_set, mnistm_train_set):
        dl, tl = train_step(sx, tx)
        dll.append(dl.numpy())
        tll.append(tl.numpy())
        
    if (epoch + 1) % 10 == 0:
        tgt_opt.learning_rate = tgt_opt.learning_rate/2
        disc_opt.learning_rate = disc_opt.learning_rate/2
        checkpoint = tf.train.Checkpoint(target_optimizer=tgt_opt,
                                         discriminator_optimizer=disc_opt,
                                         source_encoder=source_encoder,
                                         target_encoder=target_encoder,
                                         source_classifier=source_classifier,
                                         ciritic=critic)
        checkpoint.save(file_prefix = checkpoint_prefix)
    
    print("Epoch:"+str(epoch), end=" => ")
    model_eval(mnistm_test_set, target_encoder)
#     print("Current lr: {}".format(tgt_opt.learning_rate))
plt.plot(dll)
plt.plot(tll)
model_eval(mnistm_train_set, source_encoder)
model_eval(mnistm_train_set, target_encoder)
model_eval(mnistm_test_set, source_encoder)
model_eval(mnistm_test_set, target_encoder)<jupyter_output>Loss: 2.014 | Accuracy: 38.93%
Loss: 4.587 | Accuracy: 64.96%
{'fraction_non_alphanumeric': 0.11570469798657718, 'fraction_numerical': 0.018389261744966443, 'mean_word_length': 3.0738108255877528, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '88213', 'license': 'MIT', 'n_tokens_mistral': 2632, 'n_tokens_neox': 2508, 'n_words': 420, 'path': 'adda_usps.ipynb', 'repo_name': 'jaehwi0823/ADDA', 'size': '7450'}