<jupyter_start><jupyter_text>Gastric Cancer VS Cancer<jupyter_code>gc_c = pd.concat([gastric_df, cancer_df]).sample(frac=1, random_state=seed_value)
gc_c
doc_lens = []
for i, row in gc_c.iterrows():
    d_len = len(row['Document'].fulltext_string)
    doc_lens.append(d_len)

import matplotlib.pyplot as plt 
plt.hist(doc_lens)
plt.show()
#Parameters
dl_config.padding = 'post'           
dl_config.truncating = 'post'        

dl_config.epochs = 3         # recommended number of epochs: 2, 3, 4 
dl_config.batch_size = 16     # recommended batch-size: 16 or 32 
dl_config.learning_rate = 2e-5   # recommended learning rate for Adam: 5e-5, 3e-5, 2e-5   # 3e-4, 1e-4,

dl_config.max_sent_len = 512      #sentences will have a maximum of "max_sent_len" words
dl_config.nmr_sentences = 1      #[1 or 2]

dl_config.validation_percentage = 20<jupyter_output><empty_output><jupyter_text>Preprocessing  Bert<jupyter_code>dl_config.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

dl_config.tokenizer.convert_tokens_to_ids("[SEP]")
x_train, y_train, x_val, y_val =  Bert_preprocessing(gc_c, gc_c['Label'], 
                                                     dl_config, 
                                                     nmr_sentences = dl_config.nmr_sentences, 
                                                     validation_percentage = dl_config.validation_percentage, 
                                                     seed_value=dl_config.seed_value)
len(x_train[0][0])
dl_config.tokenizer.convert_ids_to_tokens(x_train[0][0])
dl_config.tokenizer.convert_ids_to_tokens(x_val[0][0])<jupyter_output><empty_output><jupyter_text>Deep Learning<jupyter_code>from mlearning.dl_models import Bert_Dense_opt
bert_name = "bert-base-uncased"
if multiple_gpus:
    with strategy.scope():
        model = Bert_Dense(dl_config, learning_rate=dl_config.learning_rate,static_bert=False, bert_name_or_path=bert_name)
        #model = Bert_FE(dl_config, learning_rate=dl_config.learning_rate, bert_name_or_path=bert_name)
        #model = Bert_Sequence_FT(dl_config, learning_rate=dl_config.learning_rate, bert_name_or_path=bert_name)
        #model = Bert_Sequence_FE(dl_config, learning_rate=dl_config.learning_rate, bert_name_or_path=bert_name)
        
else:
    model = Bert_Dense(dl_config, learning_rate=dl_config.learning_rate, static_bert=False, bert_name_or_path=bert_name)
    #model = Bert_FE(dl_config, learning_rate=dl_config.learning_rate, bert_name_or_path=bert_name)
    #model = Bert_Sequence_FT(dl_config, learning_rate=dl_config.learning_rate, bert_name_or_path=bert_name)
    #model = Bert_Sequence_FE(dl_config, learning_rate=dl_config.learning_rate, bert_name_or_path=bert_name)
dl_config.keras_callbacks = False

if dl_config.keras_callbacks:
    dl_config.patience = 2   #early-stopping patience
    checkpoint_path = str(dl_config.model_id_path) + '/checkpoint.hdf5'
    keras_callbacks = [
            EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=dl_config.patience),
            ModelCheckpoint(checkpoint_path, monitor='val_loss', mode='min', verbose=1, save_best_only=True)
    ]
else:
    keras_callbacks=None
history = model.fit(x_train, y_train,
                    epochs=dl_config.epochs,
                    batch_size=dl_config.batch_size,
                    validation_data=(x_val, y_val),
                    callbacks=keras_callbacks)

if dl_config.keras_callbacks:
    model.load_weights(checkpoint_path)<jupyter_output>Epoch 1/3
WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.
WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.
WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.
INFO:tensorflow:batch_all_reduce: 198 all-reduces with algorithm = nccl, num_packs = 1
WARNING:tensorflow:Efficient allreduce is not supported for 3 IndexedSlices
INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:GPU:1 then broadcast to ('/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2', '/job:localhost/replica:0/task:0/device:GPU:3').
INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:GPU:1 then broadcast[...]<jupyter_text>Evaluation<jupyter_code>#train_loss, dl_config.train_acc = model.evaluate(x_train, y_train, verbose=0)
#print('Training Loss: %.3f' % (train_loss))
#print('Training Accuracy: %.3f' % (dl_config.train_acc))


plot_training_history(history_dict = history, dl_config=dl_config)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10623317457030441, 'fraction_numerical': 0.01490991923793746, 'mean_word_length': 4.4391891891891895, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '136868', 'license': 'MIT', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1666, 'n_words': 252, 'path': 'pipelines/Gastric_Cancer.ipynb', 'repo_name': 'BioSystemsUM/biotmpy', 'size': '4829'}