<jupyter_start><jupyter_text>THE NEXT STEP TAKES AROUND 15 HOURS TO COMPLETE WITH 13GB DATA AND HAS TO BE RUN PRIOR TO ANY PRETRAINING<jupyter_code>#docker exec -w /bachelor_project/electra_google gpu_0_electra python3 -u build_pretraining_dataset.py --corpus-dir /bachelor_project/data/training_data --vocab-file /bachelor_project/models/ælæctra_cased_32k/vocab.txt --output-dir /bachelor_project/models/ælæctra_cased_32k/pretrain_tfrecords --max-seq-length 128 --blanks-separate-docs False --no-lower-case --num-processes 6<jupyter_output><empty_output><jupyter_text>!python3 build_pretraining_dataset.py \  --corpus-dir $DATA_DIR \  --vocab-file $DATA_DIR/vocab.txt \  --output-dir $DATA_DIR/pretrain_tfrecords \  --max-seq-length 128 \  --blanks-separate-docs False \  --no-lower-case \  --num-processes 6<jupyter_code>hparams = {
    "do_train": "true",
    "do_eval": "false",
    "model_size": "small",
    "do_lower_case": "false",
    "vocab_size": 32000,
    "num_train_steps": 1e6,
    "save_checkpoints_steps": 100000,
    "train_batch_size": 128,
}
           
with open("/bachelor_project/models/ælæctra_cased_32k/hparams.json", "w") as f:
    json.dump(hparams, f)
#docker exec -w /bachelor_project gpu_1_electra python3 -u /bachelor_project/electra_google/run_pretraining.py --data-dir /bachelor_project/models/ælæctra_cased_32k --model-name ælæctra_cased_32k --hparams "/bachelor_project/models/ælæctra_cased_32k/hparams.json"
#tensorboard --logdir=/bachelor_project/models/ælæctra_cased_32k/models/ælæctra_cased_32k --host localhost --port 8089<jupyter_output><empty_output><jupyter_text>!python3 run_pretraining.py \  --data-dir $DATA_DIR \  --model-name $MODEL_NAME \  --hparams "hparams.json" !git clone https://github.com/lonePatient/electra_pytorch.git<jupyter_code>MODEL_DIR = "/bachelor_project/models/ælæctra_cased_32k/models/ælæctra_cased_32k/"

config = {
  "vocab_size": 32000,
  "embedding_size": 128,
  "hidden_size": 256,
  "num_hidden_layers": 12,
  "num_attention_heads": 4,
  "intermediate_size": 1024,
  "generator_size": "0.25",
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "attention_probs_dropout_prob": 0.1,
  "max_position_embeddings": 512,
  "type_vocab_size": 2,
  "initializer_range": 0.02
}

with open(MODEL_DIR + "config.json", "w") as f:
    json.dump(config, f)
!python electra_pytorch/convert_electra_tf_checkpoint_to_pytorch.py \
    --tf_checkpoint_path=$MODEL_DIR \
    --electra_config_file=$MODEL_DIR/config.json \
    --pytorch_dump_path=$MODEL_DIR/pytorch_model.bin
import torch
from transformers import ElectraForPreTraining, ElectraTokenizerFast

discriminator = ElectraForPreTraining.from_pretrained(MODEL_DIR)
tokenizer = ElectraTokenizerFast.from_pretrained(MODEL_DIR, do_lower_case=False)

sentence = "Fuglene synger" # The birds are singing
fake_sentence = "<NAME>" # The birds are speaking 

fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True)
fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt")
discriminator_outputs = discriminator(fake_inputs)
predictions = discriminator_outputs[0] > 0

[print("%7s" % token, end="") for token in fake_tokens]
print("\n")
[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()];
discriminator.num_parameters()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12654787073391724, 'fraction_numerical': 0.028390214436726065, 'mean_word_length': 5.481409001956947, 'pattern_counts': {'":': 21, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '431661', 'license': 'MIT', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1260, 'n_words': 201, 'path': 'notebooks/pretraining/train_cased_32k_danish_electra.ipynb', 'repo_name': 'MalteHB/-l-ctra', 'size': '3318'}