<jupyter_start><jupyter_text>Benchmark ProtElectra Model using GPU or CPU 1. Load necessry libraries including huggingface transformers<jupyter_code>!pip install -q transformers
import torch
from transformers import ElectraForMaskedLM, ElectraModel, pipeline
import time
from datetime import timedelta
import os
import requests
from tqdm.auto import tqdm<jupyter_output><empty_output><jupyter_text>2. Set the url location of ProtElectr and the vocabulary file<jupyter_code>generatorModelUrl = 'https://www.dropbox.com/s/5x5et5q84y3r01m/pytorch_model.bin?dl=1'
discriminatorModelUrl = 'https://www.dropbox.com/s/9ptrgtc8ranf0pa/pytorch_model.bin?dl=1'

generatorConfigUrl = 'https://www.dropbox.com/s/9059fvix18i6why/config.json?dl=1'
discriminatorConfigUrl = 'https://www.dropbox.com/s/jq568evzexyla0p/config.json?dl=1'

vocabUrl = 'https://www.dropbox.com/s/wck3w1q15bc53s0/vocab.txt?dl=1'<jupyter_output><empty_output><jupyter_text>3. Download ProtElectra models and vocabulary files<jupyter_code>downloadFolderPath = 'models/electra/'
discriminatorFolderPath = os.path.join(downloadFolderPath, 'discriminator')
generatorFolderPath = os.path.join(downloadFolderPath, 'generator')

discriminatorModelFilePath = os.path.join(discriminatorFolderPath, 'pytorch_model.bin')
generatorModelFilePath = os.path.join(generatorFolderPath, 'pytorch_model.bin')

discriminatorConfigFilePath = os.path.join(discriminatorFolderPath, 'config.json')
generatorConfigFilePath = os.path.join(generatorFolderPath, 'config.json')

vocabFilePath = os.path.join(downloadFolderPath, 'vocab.txt')
if not os.path.exists(discriminatorFolderPath):
    os.makedirs(discriminatorFolderPath)
if not os.path.exists(generatorFolderPath):
    os.makedirs(generatorFolderPath)
def download_file(url, filename):
  response = requests.get(url, stream=True)
  with tqdm.wrapattr(open(filename, "wb"), "write", miniters=1,
                    total=int(response.headers.get('content-length', 0)),
                    desc=filename) as fout:
      for chunk in response.iter_content(chunk_size=4096):
          fout.write(chunk)
if not os.path.exists(generatorModelFilePath):
    download_file(generatorModelUrl, generatorModelFilePath)

if not os.path.exists(discriminatorModelFilePath):
    download_file(discriminatorModelUrl, discriminatorModelFilePath)
    
if not os.path.exists(generatorConfigFilePath):
    download_file(generatorConfigUrl, generatorConfigFilePath)

if not os.path.exists(discriminatorConfigFilePath):
    download_file(discriminatorConfigUrl, discriminatorConfigFilePath)
    
if not os.path.exists(vocabFilePath):
    download_file(vocabUrl, vocabFilePath)<jupyter_output><empty_output><jupyter_text>4. Load Electra and generator Models<jupyter_code>electra = ElectraModel.from_pretrained(discriminatorFolderPath)
generator = ElectraForMaskedLM.from_pretrained(generatorFolderPath)<jupyter_output><empty_output><jupyter_text>5. Load the models into the GPU if avilabile<jupyter_code>device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
generator = generator.to(device)
generator = generator.eval()
electra = electra.to(device)
electra = electra.eval()<jupyter_output><empty_output><jupyter_text>6. Benchmark Configuration<jupyter_code>min_batch_size = 8
max_batch_size = 32
inc_batch_size = 8

min_sequence_length = 64
max_sequence_length = 512
inc_sequence_length = 64

iterations = 10<jupyter_output><empty_output><jupyter_text>7. Start Benchmarking Electra<jupyter_code>device_name = torch.cuda.get_device_name(device.index) if device.type == 'cuda' else 'CPU'

with torch.no_grad():
    print((' Benchmarking using ' + device_name + ' ').center(80, '*'))
    print(' Start '.center(80, '*'))
    for sequence_length in range(min_sequence_length,max_sequence_length+1,inc_sequence_length):
        for batch_size in range(min_batch_size,max_batch_size+1,inc_batch_size):
            start = time.time()
            for i in range(iterations):
                input_ids = torch.randint(1, 20, (batch_size,sequence_length)).to(device)
                results = electra(input_ids)[0].cpu().numpy()
            end = time.time()
            ms_per_protein = (end-start)/(iterations*batch_size)
            print('Sequence Length: %4d \t Batch Size: %4d \t Ms per protein %4.2f' %(sequence_length,batch_size,ms_per_protein))
        print(' Done '.center(80, '*'))
    print(' Finished '.center(80, '*'))<jupyter_output>******************* Benchmarking using Tesla P100-PCIE-16GB ********************
************************************ Start *************************************
Sequence Length:   64 	 Batch Size:    8 	 Ms per protein 0.01
Sequence Length:   64 	 Batch Size:   16 	 Ms per protein 0.01
Sequence Length:   64 	 Batch Size:   24 	 Ms per protein 0.01
Sequence Length:   64 	 Batch Size:   32 	 Ms per protein 0.01
************************************* Done *************************************
Sequence Length:  128 	 Batch Size:    8 	 Ms per protein 0.02
Sequence Length:  128 	 Batch Size:   16 	 Ms per protein 0.02
Sequence Length:  128 	 Batch Size:   24 	 Ms per protein 0.02
Sequence Length:  128 	 Batch Size:   32 	 Ms per protein 0.02
************************************* Done *************************************
Sequence Length:  192 	 Batch Size:    8 	 Ms per protein 0.02
Sequence Length:  192 	 Batch Size:   16 	 Ms per protein 0.02
Sequence Length:  192 	 Batch Size:   24 	 Ms [...]<jupyter_text>7. Start Benchmarking Generator<jupyter_code>device_name = torch.cuda.get_device_name(device.index) if device.type == 'cuda' else 'CPU'

with torch.no_grad():
    print((' Benchmarking using ' + device_name + ' ').center(80, '*'))
    print(' Start '.center(80, '*'))
    for sequence_length in range(min_sequence_length,max_sequence_length+1,inc_sequence_length):
        for batch_size in range(min_batch_size,max_batch_size+1,inc_batch_size):
            start = time.time()
            for i in range(iterations):
                input_ids = torch.randint(1, 20, (batch_size,sequence_length)).to(device)
                results = generator(input_ids)[0].cpu().numpy()
            end = time.time()
            ms_per_protein = (end-start)/(iterations*batch_size)
            print('Sequence Length: %4d \t Batch Size: %4d \t Ms per protein %4.2f' %(sequence_length,batch_size,ms_per_protein))
        print(' Done '.center(80, '*'))
    print(' Finished '.center(80, '*'))<jupyter_output>******************* Benchmarking using Tesla P100-PCIE-16GB ********************
************************************ Start *************************************
Sequence Length:   64 	 Batch Size:    8 	 Ms per protein 0.00
Sequence Length:   64 	 Batch Size:   16 	 Ms per protein 0.00
Sequence Length:   64 	 Batch Size:   24 	 Ms per protein 0.00
Sequence Length:   64 	 Batch Size:   32 	 Ms per protein 0.00
************************************* Done *************************************
Sequence Length:  128 	 Batch Size:    8 	 Ms per protein 0.00
Sequence Length:  128 	 Batch Size:   16 	 Ms per protein 0.00
Sequence Length:  128 	 Batch Size:   24 	 Ms per protein 0.00
Sequence Length:  128 	 Batch Size:   32 	 Ms per protein 0.00
************************************* Done *************************************
Sequence Length:  192 	 Batch Size:    8 	 Ms per protein 0.00
Sequence Length:  192 	 Batch Size:   16 	 Ms per protein 0.00
Sequence Length:  192 	 Batch Size:   24 	 Ms [...]
{'fraction_non_alphanumeric': 0.1654347533933611, 'fraction_numerical': 0.03547910227120011, 'mean_word_length': 4.68960244648318, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 5, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '525830', 'license': 'AFL-3.0', 'n_tokens_mistral': 2541, 'n_tokens_neox': 2296, 'n_words': 539, 'path': 'Benchmark/ProtElectra.ipynb', 'repo_name': 'taneishi/ProtTrans', 'size': '7441'}