<jupyter_start><jupyter_text>Inital Mucking Around with BERTTo add additional vocab, see https://github.com/google-research/bert/issues/9<jupyter_code>#place unzipped BERT model files in same level dir
#https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip

import os
UPLOAD_TIME = '2018_10_18' #@param {type:"string"}
BERT_MODEL = 'uncased_L-12_H-768_A-12' #@param {type:"string"}
BERT_PRETRAINED_DIR = os.path.realpath(BERT_MODEL)
print('***** BERT pretrained directory: {} *****'.format(BERT_PRETRAINED_DIR))
import os
import json
import keras 
from keras_bert.loader import load_trained_model_from_checkpoint
from keras_bert.bert import *
config_file = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')
checkpoint_file = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')
model = load_trained_model_from_checkpoint(config_file, checkpoint_file, training=True)

#model = keras.models.load_model('../Train model/weights.11-3.73.hdf5')

model.summary(line_length=120)
model.compile('adam', 'sparse_categorical_crossentropy')
dict_path = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')
dict_path
import codecs

bsz = 8

MASK = ['[MASK]']
CLS = ['[CLS]']
SEP = ['[SEP]']
UNK = ['[UNK]']

tokens = CLS + 'VALVE'.lower().split() + 'GLOBE LESS EQ'.lower().split() + MASK + 'SEAL 0.75 IN 2173# \
PER ANSI B16.34-1981 FORGED SSBODY BOLTED BONNET, SOCKET WELD'.lower().split() + SEP + 'MUST COMPLY WITH ASME B&PV CODE, \
SECT III, CL 2, 1974 ED W75 ADD AND SPEC CMTR FOR BODY (SA182 TP 316L), INDICATOR TUBE (SA479 GR 316), \
BONNET TUBE (SA479 GR XM-19), MAIN DISC (SA564 GR 630), AND FIXED CORE (A276 GR 410 OR A479 GR 410), \
NOREM 02A HARD FACING'.lower().split() + SEP + 'MUST PROVIDE CERTIFICATION TO EQUIP QUALIFICATION AND SEISMIC REQUIREMENTS PER \
SPEC, PT/MT FOR HARD FACING/OVERLAY'.lower().split() + SEP

#develop
tokens = CLS + 'Attempting to'.lower().split() + MASK + 'inventions he could patent and market, \
Tesla conducted a range of experiments with mechanical oscillators generators \
electrical discharge tubes and early X-ray imaging'.lower().split() + SEP + \
'He also built a wireless controlled boat , one of the first ever exhibited'.lower().split() + SEP

#cleared
tokens = CLS + 'ON 01/04/2019 at 01:29 The Control Room received alarm window 5B02D , PPS TROUBLE / GND ,\
and RJ point SBYS36, Plant Protection System Channel D Ground. The alarm locked in for 45 minutes \
then'.lower().split() + MASK +\
'. The alarm response was followed and no abnormalities were identified'.lower().split() + SEP + 'The ground alarm light on the\
front of PPS Cabinet D is ON which is normal , meaning there is no ground indicated'.lower().split() + SEP + 'Wrote CR to\
request support from I&C'.lower().split() + SEP

#cleared, None:There was no word there
tokens = CLS + 'ON 01/04/2019 at 01:29 The Control Room received alarm window 5B02D , PPS TROUBLE / GND ,\
and RJ point SBYS36, Plant Protection System Channel D Ground. The alarm locked in for 45 minutes \
then'.lower().split() + MASK +\
'. The alarm response was followed and no'.lower().split() + MASK +  'abnormalities were identified'.lower().split() + SEP + 'The ground alarm light on the\
front of PPS Cabinet D is ON which is normal , meaning there is no ground indicated'.lower().split() + SEP + 'Wrote CR to\
request support from I&C'.lower().split() + SEP

#cleared, abnormalities
tokens = CLS + 'ON 01/04/2019 at 01:29 The Control Room received alarm window 5B02D , PPS TROUBLE / GND ,\
and RJ point SBYS36, Plant Protection System Channel D Ground. The alarm locked in for 45 minutes \
then'.lower().split() + MASK +\
'. The alarm response was followed and no'.lower().split() + MASK +  'were identified'.lower().split() + SEP + 'The ground alarm light on the\
front of PPS Cabinet D is ON which is normal , meaning there is no ground indicated'.lower().split() + SEP + 'Wrote CR to\
request support from I&C'.lower().split() + SEP

print(tokens)

token_dict = {}
with codecs.open(dict_path, 'r', 'utf8') as reader:
    for line in reader:
        token = line.strip()
        token_dict[token] = len(token_dict)
token_dict_rev = {v: k for k, v in token_dict.items()}

def token_lookup(token):
    try:
        x = token_dict[token]
        return x
    except KeyError:
        return token_dict['[UNK]']
    
token_input = np.asarray([[token_lookup(token) for token in tokens] + [0] * (512 - len(tokens)) for i in range(bsz)])
seg_input = np.asarray([[0] * len(tokens) + [0] * (512 - len(tokens)) for i in range(bsz)])
mask_input = np.asarray([[0, 1, 1] + [0] * (512 - 3) for i in range(bsz)])
print(token_input[0][:len(tokens)])
predicts = model.predict([token_input, seg_input, mask_input])[0]
predicts = np.argmax(predicts, axis=-1)
print(predicts[0][:len(tokens)])
print(list(map(lambda x: token_dict_rev[x], predicts[0][1:len(tokens)])))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1339066339066339, 'fraction_numerical': 0.03705978705978706, 'mean_word_length': 4.614942528735632, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '115917', 'license': 'MIT', 'n_tokens_mistral': 1832, 'n_tokens_neox': 1674, 'n_words': 537, 'path': 'getting_it_running/explore_example_code.ipynb', 'repo_name': 'bradfox2/keras-bert', 'size': '4884'}