<jupyter_start><jupyter_text>BioMedBERT - BREATHE using BERT Weights and Vocab<jupyter_code># !sudo pip install tensorflow==1.15
import os
import sys
import json
import tensorflow as tf
tf.__version__<jupyter_output><empty_output><jupyter_text>Save model assets and checkpoints to GCS<jupyter_code>BUCKET_NAME = "ekaba-assets"
MODEL_DIR = "biomedbert_base_bert_weights_and_vocab"
tf.io.gfile.mkdir(MODEL_DIR)<jupyter_output><empty_output><jupyter_text>Hyparameter configuration for BERT BASE<jupyter_code># VOC_SIZE = 32000
# VOC_FNAME = "biomedbert-8M.txt"
# # use this for BERT-base

# bert_base_config = {
#     "attention_probs_dropout_prob": 0.1,
#     "directionality": "bidi",
#     "hidden_act": "gelu",
#     "hidden_dropout_prob": 0.1,
#     "hidden_size": 768,
#     "initializer_range": 0.02,
#     "intermediate_size": 3072,
#     "max_position_embeddings": 512,
#     "num_attention_heads": 12,
#     "num_hidden_layers": 12,
#     "pooler_fc_size": 768,
#     "pooler_num_attention_heads": 12,
#     "pooler_num_fc_layers": 3,
#     "pooler_size_per_head": 128,
#     "pooler_type": "first_token_transform",
#     "type_vocab_size": 2,
#     "vocab_size": VOC_SIZE
# }

# with open("{}/bert_config.json".format(MODEL_DIR), "w") as fo:
#     json.dump(bert_base_config, fo, indent=2)
# update vocab
# !cp ../vocabulary/full_text/biomedbert-8M.txt \
# biomedbert_base_bert_weights_breathe_vocab/biomedbert-8M.txt
# %%bash
# cd biomedbert_base_bert_weights_and_vocab
# wget https://storage.googleapis.com/bert_models/2019_05_30/wwm_cased_L-24_H-1024_A-16.zip
# %%bash
# cd biomedbert_base_bert_weights_and_vocab
# unzip wwm_cased_L-24_H-1024_A-16.zip
# %%bash
# mv biomedbert_base_bert_weights_and_vocab/wwm_cased_L-24_H-1024_A-16/* biomedbert_base_bert_weights_and_vocab
# !rm -rf biomedbert_base_bert_weights_and_vocab/wwm_cased_L-24_H-1024_A-16.zip
# !du biomedbert_base_bert_weights_and_vocab/* -h
# # move to GCS
# !gsutil -m cp -r $MODEL_DIR gs://ekaba-assets/
# tf.io.gfile.glob('gs://ekaba-assets/pre_trained_data_full_biomed/*')
init_checkpoint = tf.train.latest_checkpoint('gs://ekaba-assets/biomedbert_base_bert_weights_and_vocab')
init_checkpoint
%%bash
export INPUT_FILE=gs://ekaba-assets/pre_trained_breathe_bert_vocab/shard_*
export OUTPUT_DIR=gs://ekaba-assets/biomedbert_base_bert_weights_and_vocab
export BERT_BASE_DIR=gs://ekaba-assets/biomedbert_base_bert_weights_and_vocab

python3 bert/run_pretraining.py \
  --input_file="$INPUT_FILE" \
  --output_dir="$OUTPUT_DIR" \
  --do_train=True \
  --do_eval=True \
  --bert_config_file="$BERT_BASE_DIR"/bert_config.json \
  --init_checkpoint="$BERT_BASE_DIR"/bert_model.ckpt \
  --train_batch_size=256 \
  --eval_batch_size=256 \
  --max_seq_length=128 \
  --max_predictions_per_seq=20 \
  --num_train_steps=10000000 \
  --num_warmup_steps=1000 \
  --learning_rate=1e-5 \
  --use_tpu=true \
  --tpu_name=for-shweta-tpu \
  --tpu_zone=europe-west4-a \
  --gcp_project=ai-vs-covid19 \
  --num_tpu_cores=128<jupyter_output>Process is interrupted.
{'fraction_non_alphanumeric': 0.1317880794701987, 'fraction_numerical': 0.0380794701986755, 'mean_word_length': 5.15274949083503, 'pattern_counts': {'":': 17, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '198585', 'license': 'MIT', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1255, 'n_words': 154, 'path': 'notebooks/Model_BERT_Weights_Vocab.ipynb', 'repo_name': 'BioMedBERT/biomedbert', 'size': '3020'}