<jupyter_start><jupyter_text>**Load Model Weights** : ---*   Those weights are generated from the notebook ***transformers_truncated_baseline.ipynb*** *   I've uploaded the model weights into my drive . *   here is the link for the Model Weights : **[transformer Truncated weights fold0](https://drive.google.com/drive/folders/1rMpwkqPf8JEQ-1il1xMCEOZTGe5yZDeC?usp=sharing)*** To speed the process you can **add a shortcut** to your drive and run the following code<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!cp '/content/drive/MyDrive/Code Review /Transformers Truncated Weights/training_folds_0.ckpt.data-00000-of-00001' .
!cp '/content/drive/MyDrive/Code Review /Transformers Truncated Weights/training_folds_0.ckpt.index' .
train = pd.read_csv('/content/drive/MyDrive/Code Review /Train.csv')
train.head()
test = pd.read_csv('/content/drive/MyDrive/Code Review /Test.csv')
test.head()
small_labels = train['Label'].value_counts().reset_index().loc[15:, 'index'].tolist()
small_labels += ['ECONOMY', "SOCIAL ISSUES", 'SOCIAL']
print(small_labels)
train_small = train[train['Label'].isin(small_labels)].reset_index(drop=True)
train_large = train[~train['Label'].isin(small_labels)].reset_index(drop=True)
LB = LabelEncoder()
train_small['Label'] = LB.fit_transform(train_small['Label'])
############################################################################################################

########################################### " SEED HERE " #################################################

seed_everything(seed=0)
AUTO = tf.data.experimental.AUTOTUNE
# Configuration
EPOCHS = 30
N_LABELS = train_small['Label'].unique().shape[0] 
BATCH_SIZE = 32
############################################################################################################

########################################### " SEED HERE " #################################################

seed_everything(seed=0)
df = pd.concat((train, test))
dataset = tf.data.Dataset.from_tensor_slices(df['Text'].values)
############################################################################################################

########################################### " SEED HERE " ##################################################

seed_everything(seed=0)
vocab_size = 100000
maxlen = 100
encoder = tf.keras.layers.experimental.preprocessing.TextVectorization(
    max_tokens=vocab_size, output_sequence_length=maxlen)
encoder.adapt(dataset)
%%time 
def reformat(x, y):
    return x, tf.cast(y, tf.float32)

def df_to_dataset(data, labels, data_type='Train'):
    x_token = data['Text'].values
    if data_type=='Train':
        y_label = labels.values
        dataset = (tf.data.Dataset
                    .from_tensor_slices((x_token, y_label))
                    .repeat()
                    .shuffle(2048)
                    .batch(BATCH_SIZE)
                    .prefetch(AUTO))
        dataset = dataset.map(reformat)
    elif data_type=='Val':
        y_label = labels.values
        dataset = (  tf.data.Dataset
                    .from_tensor_slices((x_token, y_label))
                    .batch(BATCH_SIZE)
                    .cache()
                    .prefetch(AUTO)
                     ) 
        dataset =dataset.map(reformat)
    else:
        dataset = (tf.data.Dataset
                    .from_tensor_slices(x_token)
                    .batch(BATCH_SIZE)
                    )
    return dataset
class TokenAndPositionEmbedding(layers.Layer):
    def __init__(self, maxlen, vocab_size, embed_dim):
        super(TokenAndPositionEmbedding, self).__init__()
        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)
        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)

    def call(self, x):
        maxlen = tf.shape(x)[-1]
        positions = tf.range(start=0, limit=maxlen, delta=1)
        positions = self.pos_emb(positions)
        x = self.token_emb(x)
        return x + positions
    

class TransformerBlock(layers.Layer):
    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.6):
        super(TransformerBlock, self).__init__()
        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)
        self.ffn = keras.Sequential(
            [layers.Dense(ff_dim, activation="relu"), layers.Dense(embed_dim),]
        )
        self.layernorm1 = layers.LayerNormalization()
        self.layernorm2 = layers.LayerNormalization()
        self.dropout1 = layers.Dropout(rate)
        self.dropout2 = layers.Dropout(rate)

    def call(self, inputs, training):
        attn_output = self.att(inputs, inputs)
        attn_output = self.dropout1(attn_output, training=training)
        out1 = self.layernorm1(inputs + attn_output)
        ffn_output = self.ffn(out1)
        ffn_output = self.dropout2(ffn_output, training=training)
        return self.layernorm2(out1 + ffn_output)

def BERTModel(embed_dim = 130, num_heads = 6,  ff_dim = 32):
    ############################################################################################################

    ########################################### " SEED HERE " ##################################################
    
    seed_everything(seed=0)
    inputs = layers.Input(shape=(),dtype=tf.string)
    x = encoder(inputs)
    embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)
    x = embedding_layer(x)
    transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
    x = transformer_block(x)
    x = layers.GlobalAveragePooling1D()(x)
    x = tf.keras.layers.BatchNormalization()(x)
    x = tf.keras.layers.Dense(256, activation = "relu")(x) 
    x = tf.keras.layers.Dropout(0.15)(x)
    x = tf.keras.layers.BatchNormalization()(x)
    x = tf.keras.layers.Dense(128, activation = "sigmoid")(x) 
    x = tf.keras.layers.Dropout(0.4)(x) 
    outputs = layers.Dense(N_LABELS, activation="sigmoid")(x)
    model = keras.Model(inputs=inputs, outputs=outputs)     
    return model 
def build_classifier():
    ############################################################################################################

    ########################################### " SEED HERE " ##################################################
    seed_everything(seed=0)
    model = BERTModel()    
    ############################################################################################################
    checkpoint_path = "training_folds_0.ckpt"
    model.load_weights(checkpoint_path)
    # Freeze layers execpt last threee layers
    for layer in model.layers[:-3]: 
        layer.trainable=False      
    # Define Loss
    losses = tf.keras.losses.CategoricalCrossentropy( from_logits=True)
    # compile all
    model.compile(tf.keras.optimizers.Adam(1e-7), loss=losses, metrics=["accuracy"])    
    return model

def get_model():
    with strategy.scope():
        ############################################################################################################

        ########################################### " SEED HERE " ##################################################
        seed_everything(seed=0)
        model = build_classifier()
    return model
LABEL = 'Label'
n_splits = 5

kf = StratifiedKFold(n_splits=n_splits, random_state=47, shuffle=True)

y_oof = np.zeros([train_small.shape[0], N_LABELS])
y_large = np.zeros([train_large.shape[0], N_LABELS])
y_test = np.zeros([test.shape[0], N_LABELS])


############################################################################################################

########################################### " SEED HERE " ###########################################
seed_everything(seed=0)
train_large_ds =  df_to_dataset(train_large, labels=None, data_type='Test')
test_ds = df_to_dataset(test,labels=None,data_type='Test')

i = 0
metrics = list()

y_train = pd.get_dummies(train_small['Label'])

for tr_idx, val_idx in kf.split(train_small[['Text']], train_small['Label']):

    ############################################################################################################

    ########################################### " SEED HERE " ###########################################
    seed_everything(seed=0)
    df_tr = train_small.iloc[tr_idx, :]
    df_vl = train_small.iloc[val_idx, :]
    
    tr_ds = df_to_dataset(df_tr,y_train.iloc[tr_idx, :], data_type='Train')
    vl_ds = df_to_dataset(df_vl, y_train.iloc[val_idx, :],data_type='Val')
        
    
    model =  get_model()
    
    # Include the epoch in the file name (uses `str.format`)
    checkpoint_path = f"training/training_folds_{i}.ckpt"
    checkpoint_dir = os.path.dirname(checkpoint_path)
    
    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_path,
    save_weights_only=True,
    monitor='val_accuracy',
    mode='max',
    save_best_only=True)
 
      
    # Train the model  
    n_steps = df_tr.shape[0] // BATCH_SIZE
    train_history = model.fit(
        tr_ds,
        steps_per_epoch=n_steps,
        validation_data = vl_ds,
        epochs=5, callbacks=[model_checkpoint_callback]
    )
    
    # The model weights (that are considered the best) are loaded into the model.
    model.load_weights( checkpoint_path)
    y_pred = model.predict(vl_ds.map(lambda x,y:x))
    y_oof[val_idx, :] = y_pred
    y_vl = train_small['Label'].iloc[val_idx]   
    metric = accuracy_score(y_vl, np.argmax(y_pred, 1))
    print("fold #{} val_loss: {}".format(i, metric))

    # Make predictions of rest of train the model
    proba_large =  model.predict(train_large_ds)
    y_large += proba_large / n_splits
    
    # Make predictions with the model
    y_test += model.predict(test_ds) / n_splits
    metrics.append(metric)
    i += 1


metrics = np.array(metrics).mean()
print(f'Full accuracy {metrics}')  # 0.78 
tmp_train_small = pd.DataFrame(y_oof)
tmp_train_small = pd.concat((train_small['ID'], tmp_train_small), 1)
tmp_train_large = pd.DataFrame(y_large)
tmp_train_large = pd.concat((train_large['ID'], tmp_train_large), 1)

tmp_train = train[['ID']].merge(pd.concat((tmp_train_small, tmp_train_large)), on='ID', how='left')
tmp_train = tmp_train.drop('ID', 1)
tmp_train.to_csv(f'DNN_truncated_train.csv', index=False)
tmp_test = pd.DataFrame(y_test)
tmp_test.to_csv(f'DNN_truncated_test.csv', index=False)
!cp /content/DNN_truncated_test.csv '/content/drive/MyDrive/Code Review '
!cp /content/DNN_truncated_train.csv '/content/drive/MyDrive/Code Review '<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.266590714896041, 'fraction_numerical': 0.009778790468052787, 'mean_word_length': 4.317516405855629, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '597162', 'license': 'MIT', 'n_tokens_mistral': 3354, 'n_tokens_neox': 3177, 'n_words': 528, 'path': 'Competition-Solutions/Text/AI4D Malawi News Classification Challenge/Solution 1/transformers_truncated_baseline_ckpt.ipynb', 'repo_name': 'ZindiAfrica/Natural-Language-Processing-NLP-', 'size': '10533'}