<jupyter_start><jupyter_text>start bert as a service in terminalpython C:/Users/peter/Desktop/Capstone/hw/Data_Folder/start-bert-as-service.py -model_dir C:/Users/peter/Desktop/Capstone/hw/Data_Folder/uncased_L-12_H-768_A-12/ -num_worker=1 parallelize on cpu instead of running on gpustart bert as a service in terminal with biobertpython C:/Users/peter/Desktop/Capstone/hw/Data_Folder/start-bert-as-service.py -model_dir C:/Users/peter/Desktop/Capstone/hw/Data_Folder/uncased_L-12_H-768_A-12-bio/ -num_worker=7 -max_seq_len=50 -cpu -max_batch_size 16 run on gpustart bert as a service in terminal with biobertpython C:/Users/peter/Desktop/Capstone/hw/Data_Folder/start-bert-as-service.py -model_dir C:/Users/peter/Desktop/Capstone/hw/Data_Folder/uncased_L-12_H-768_A-12-bio/ -max_seq_len=50 -num_worker=1<jupyter_code>#is this sufficient for submission or should it be on the git or something?
#biobert v1.1
#https://github.com/naver/biobert-pretrained
#BioBERT-Base v1.1 (+ PubMed 1M) - based on BERT-base-Cased (same vocabulary)
#uncased_L-12_H-768_A-12-bio
#once terminal says "ready to serve request", connect
#may not be able to complete execution if the server isn't up
#avoid this because you might not be able to interrupt
bc = BertClient()
#if want to close connection, run bc.close()
#even after terminating it will stay open if you don't close 
bc.close()
#shut down server, don't forget
!bert-serving-terminate -port 5555
#can check client status with
bc.status
#can check server status with
bc.server_status
bc.encode(['hello world!', 'thisis it'])
#sentence split all documents
sentences = []
for i in range(len(raw['note'])):
    sentences.append(tokenize.sent_tokenize(raw['note'][i]))

raw['sent'] = sentences
#gpu timetest
start = datetime.now()

list_of_sentembs = []

for i in range(20):
    list_of_sentembs.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
#cpu timetest
start = datetime.now()

list_of_sentembs = []

for i in range(20):
    list_of_sentembs.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
len(list_of_sentembs_0)
#first 200
start = datetime.now()

list_of_sentembs_0 = []

for i in range(0,200):
    list_of_sentembs_0.append(bc.encode(raw['sent'][i]))
    print(i)
    
with open("sentembs_0.pkl", "wb") as fp:   #Pickling
    pickle.dump(list_of_sentembs_0, fp)
    
end = datetime.now()

end - start
#second 200
start = datetime.now()

list_of_sentembs_1 = []

for i in range(200,400):
    list_of_sentembs_1.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
with open("sentembs_1.pkl", "wb") as fp:   #Pickling
    pickle.dump(list_of_sentembs_1, fp)
#third 200
start = datetime.now()

list_of_sentembs_2 = []

for i in range(400,600):
    list_of_sentembs_2.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
with open("sentembs_2.pkl", "wb") as fp:   #Pickling
    pickle.dump(list_of_sentembs_2, fp)
#fourth 200
start = datetime.now()

list_of_sentembs_3 = []

for i in range(600,800):
    list_of_sentembs_3.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
with open("sentembs_3.pkl", "wb") as fp:   #Pickling
    pickle.dump(list_of_sentembs_3, fp)
#fifth 200
start = datetime.now()

list_of_sentembs_4 = []

for i in range(800,1000):
    list_of_sentembs_4.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
with open("sentembs_4.pkl", "wb") as fp:   #Pickling
    pickle.dump(list_of_sentembs_4, fp)
#sixth
start = datetime.now()

list_of_sentembs_5 = []

for i in range(1000,1081):
    list_of_sentembs_5.append(bc.encode(raw['sent'][i]))
    print(i)
    
end = datetime.now()

end - start
with open("sentembs_5.pkl", "wb") as fp:   #Pickling
    pickle.dump(list_of_sentembs_5, fp)
len(raw)
start = datetime.now()
example = bc.encode(raw['sent'][0])
end = datetime.now()
end - start
with open("sentembs.pkl", "rb") as fp:   # Unpickling
    use = pickle.load(fp)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1392716535433071, 'fraction_numerical': 0.031496062992125984, 'mean_word_length': 4.463709677419355, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '247573', 'license': 'MIT', 'n_tokens_mistral': 1658, 'n_tokens_neox': 1570, 'n_words': 349, 'path': '01_bert_search.ipynb', 'repo_name': 'lolajohnston/deep-shallow-nlp-ensemble', 'size': '4064'}