<jupyter_start><jupyter_text>ensure jupyter notebook and bert-serving-start are running inside the virtualenv ---      BERT experiments with Imdb dataset  ---    Requirements & Config<jupyter_code>import json
import os
import random
import sys
import tensorflow as tf
import numpy as np
import pandas as pd
import pickle
from bert_serving.client import BertClient, ConcurrentBertClient
from tensorflow.estimator import BaselineClassifier
from tensorflow.python.estimator.canned.dnn import DNNClassifier
from tensorflow.python.estimator.run_config import RunConfig
from tensorflow.python.estimator.training import TrainSpec, EvalSpec, train_and_evaluate
tf.logging.set_verbosity(tf.logging.INFO)
batch_size = 128
num_parallel_calls = 1
bc = BertClient()<jupyter_output><empty_output><jupyter_text>---      Cache data---  In order to avoid having to encode embeddings for the dataset every experiment, we create separate files for BERT encodings. Creating this 'cache' is performed in chunks for practical considerations (in case of errors).<jupyter_code># pipeline
def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]

encode_count = 0
def encode(chunk):
    global encode_count
    print('Chunk {}'.format(encode_count))
    encode_count += 1
    return bc.encode(chunk)
        
def get_encodes(data):
    # x is `batch_size` of lines, each of which is a json object
    features = np.array([])
    text = [x[0] for x in data]
    features = np.concatenate([encode(chunk) for chunk in chunks(text, 256)])
    # randomly choose a label
    labels = [x[1] for x in data]
    return features, labels

def cache_data(data_dir, dest_dir, start_chunk, end_chunk):
    pos_files = os.listdir(os.path.join(data_dir, 'pos'))
    neg_files = os.listdir(os.path.join(data_dir, 'neg'))
        
    data = []
    for pos_file, neg_file in zip(pos_files, neg_files):
        with open(os.path.join(data_dir, 'pos', pos_file)) as f:
            review = f.readlines()[0].strip()
            data.append((review, 1))
        with open(os.path.join(data_dir, 'neg', neg_file)) as f:
            review = f.readlines()[0].strip()
            data.append((review, 0))
    chunk_num = -1
    chunk_size = 2048
    for chunk in chunks(data, chunk_size):
        chunk_num += 1
        if chunk_num < start_chunk:
            continue
        if chunk_num > end_chunk:
            break
        features, output = get_encodes(chunk)
        print('Wrote data_{:03d}.p'.format(chunk_num))
        with open(os.path.join(dest_dir, 'data_{:03d}.p'.format(chunk_num)), 'wb') as f:
            pickle.dump((features, output), f)<jupyter_output><empty_output><jupyter_text>BERT-Base, Uncased12-layer, 768-hidden, 12-heads, 110M parameters<jupyter_code># # !bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4
%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)<jupyter_output>Chunk 0<jupyter_text>BERT-Base, Cased12-layer, 768-hidden, 12-heads , 110M parameters<jupyter_code>%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-12_H-768_A-12/cache/train'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-12_H-768_A-12/cache/test'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)<jupyter_output>/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bert_serving/client/__init__.py:286: UserWarning: some of your sentences have more tokens than "max_seq_len=50" set on the server, as consequence you may get less-accurate or truncated embeddings.
here is what you can do:
- disable the length-check by create a new "BertClient(check_length=False)" when you do not want to display this warning
- or, start a new server with a larger "max_seq_len"
  '- or, start a new server with a larger "max_seq_len"' % self.length_limit)<jupyter_text>BERT-Large, Uncased24-layer, 1024-hidden, 16-heads, 340M parameters<jupyter_code>%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-24_H-1024_A-16/cache/train'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-24_H-1024_A-16/cache/test'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)<jupyter_output>/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bert_serving/client/__init__.py:286: UserWarning: some of your sentences have more tokens than "max_seq_len=50" set on the server, as consequence you may get less-accurate or truncated embeddings.
here is what you can do:
- disable the length-check by create a new "BertClient(check_length=False)" when you do not want to display this warning
- or, start a new server with a larger "max_seq_len"
  '- or, start a new server with a larger "max_seq_len"' % self.length_limit)<jupyter_text>BERT-Large, Cased24-layer, 1024-hidden, 16-heads, 340M parameters<jupyter_code>%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-24_H-1024_A-16/cache/train'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-24_H-1024_A-16/cache/test'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)<jupyter_output>Chunk 588<jupyter_text>preparation for testing<jupyter_code># BERT-small, uncased
dir_BSU_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train'
dir_BSU_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test'
# BERT-large, uncased
dir_BLU_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-24_H-1024_A-16/cache/train'
dir_BLU_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-24_H-1024_A-16/cache/test'
# BERT-small, cased
dir_BSC_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-12_H-768_A-12/cache/train'
dir_BSC_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-12_H-768_A-12/cache/test'
# BERT-large, cased
dir_BLC_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-24_H-1024_A-16/cache/train'
dir_BLC_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/cased_L-24_H-1024_A-16/cache/test'

# Model output directory
bert_model, classifier, input_size, train_max_steps, hidden_units, learning_rate, dropout_rate = '','','','','','',''
dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
def get_encodes(data):
    # x is `batch_size` of lines, each of which is a json object
    features = bc.encode([x[0] for x in data])
    # randomly choose a label
    labels = [x[1] for x in data]
    return features, labels

def get_input_fn(data_dir, num_examples=None, num_epochs=10):
    data_files = os.listdir(data_dir)
    if '.DS_Store' in data_files:
        data_files.remove('.DS_Store') # MAC OS cleanup
    
    # open pre-embedded data
    feature_list = []
    label_list = []
    for data_file in data_files:
        with open(os.path.join(data_dir, data_file), 'rb') as f:
            features, labels = pickle.load(f)
            feature_list.append(features)
            label_list.append(labels)
    features = np.concatenate(feature_list)
    labels = [label for labels in label_list for label in labels]
    
    # split into train and dev set
    train_features = features[0:int(0.8*len(features))]
    train_labels = labels[0:int(0.8*len(features))]
    dev_features = features[int(0.8*len(features)):len(features)]
    dev_labels = labels[int(0.8*len(features)):len(features)]
    
    train_labels = np.array(train_labels).astype('int32')
    dev_labels = np.array(dev_labels).astype('int32')
    
    if num_examples is not None:
        train_features = None #train_features[0:num_examples]
        train_labels = None #train_labels[0:num_examples]
#     print('{} train data points'.format(len(train_features)))
#     print('{} dev data points'.format(len(dev_features)))
    
    train_fn = tf.estimator.inputs.numpy_input_fn(
        x={'feature': train_features},
        y=train_labels,
        num_epochs=num_epochs,
        batch_size=128,
        shuffle=True
    )
    dev_fn = tf.estimator.inputs.numpy_input_fn(
        x={'feature': dev_features},
        y=dev_labels,
        num_epochs=1,
        batch_size=128,
        shuffle=False
    )
    return (train_fn, dev_fn)<jupyter_output><empty_output><jupyter_text>---      Expiriment: Learning Rate---  When the learning rate is too large, gradient descent can inadvertently increase rather than decrease the training error. When the learning rate is too small, training is not only slower, but may become permanently stuck with a high training error.It is not possible to calculate the best learning rate a priori. One approach to tuning this hyperparameter is to run a grid search (e.g., run with learning rates in a range like [.1, .01, .001, .0001 , .0001], and plot to see how key performance indicators change).     **uncased_L-12_H-768_A-12**  6 models  @ 0.1imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.1_dropout0.2<jupyter_code># !bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4
%%time

hidden_units = [10]
learning_rate = 0.1
bert_model = 'bert_uncased_small'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir = dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_LR1 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LR_1'])
df_LR1<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-06:30:54
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.1_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-06:30:55
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.7138, accuracy_baseline = 0.5, auc = 0.7891901, auc_precision_recall = 0.7955693, average_loss = 0.5596144, global_step = 2000, label/mean = 0.5, loss = 69.9518, precision = 0.7450[...]<jupyter_text>@ 0.05imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2<jupyter_code>%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_uncased_small'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_LR05 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LR_05'])
df_LR05<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-06:30:59
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2/model.ckpt-1563
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-06:31:00
INFO:tensorflow:Saving dict for global step 1563: accuracy = 0.7038, accuracy_baseline = 0.5, auc = 0.7782934, auc_precision_recall = 0.7903569, average_loss = 0.5676637, global_step = 1563, label/mean = 0.5, loss = 70.95796, precision = 0.74[...]<jupyter_text>@ 0.01imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.01_dropout0.2<jupyter_code>%%time

hidden_units = [10]
learning_rate = 0.01
bert_model = 'bert_uncased_small'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_LR01 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LR_01'])
df_LR01<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-06:31:01
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.01_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-06:31:02
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.7126, accuracy_baseline = 0.5, auc = 0.79420185, auc_precision_recall = 0.7977605, average_loss = 0.544821, global_step = 2000, label/mean = 0.5, loss = 68.10263, precision = 0.72[...]<jupyter_text>@ 0.001imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.001_dropout0.2<jupyter_code>hidden_units = [10]
learning_rate = 0.001
bert_model = 'bert_uncased_small'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_LR001 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LR_001'])
df_LR001<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-06:31:05
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.001_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-06:31:05
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.6904, accuracy_baseline = 0.5, auc = 0.7625588, auc_precision_recall = 0.7616379, average_loss = 0.59062976, global_step = 2000, label/mean = 0.5, loss = 73.82872, precision = 0.[...]<jupyter_text>@ 0.0001imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.0001_dropout0.2<jupyter_code>%%time

hidden_units = [10]
learning_rate = 0.0001
bert_model = 'bert_uncased_small'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_LR0001 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LR_0001'])
df_LR0001<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-06:31:08
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr0.0001_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-06:31:09
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.5888, accuracy_baseline = 0.5, auc = 0.6162612, auc_precision_recall = 0.602811, average_loss = 0.6750571, global_step = 2000, label/mean = 0.5, loss = 84.38214, precision = 0.5[...]<jupyter_text>@ 0.00001imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr1e-05_dropout0.2<jupyter_code>%%time

hidden_units = [10]
learning_rate = 0.00001
bert_model = 'bert_uncased_small'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
estimator.evaluate(dev_input_fn)
# result
df_LR00001 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LR_00001'])
df_LR00001<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-06:31:13
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_DNN_inputNone_maxsteps2000_hu10_lr1e-05_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-06:31:13
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.5112, accuracy_baseline = 0.5, auc = 0.5534019, auc_precision_recall = 0.5399217, average_loss = 0.71737605, global_step = 2000, label/mean = 0.5, loss = 89.672005, precision = 0[...]<jupyter_text>Comparison<jupyter_code>pd.concat([df_LR1, df_LR05, df_LR01, df_LR001, df_LR0001, df_LR00001], axis=1)<jupyter_output><empty_output><jupyter_text>Although `0.1` learning rate has similar accuracy to `0.01`, the AUC is higher and the average loss are lower for `Loss Rate = 0.01`. Rates closer to 0.05 or 0.01 are more typical in similar applications to avoid _gradient decent_ from passing over the global minimum and accidentally arriving at a local minimum. For many of the following experiments, we will use the `Loss Rate = 0.01` models for comparison. ---      Expiriment: Max Token Length---  By default, BERT is configured to cut sequences off at 25 tokens. While this ensures that computations don't become too large, it is also introduces an important artifact: only the first part of the reviews are considered for classification.Based on our EDA, we use `Loss Rate = 0.01`. In order to change this parameter, we* re-initialize `bert_serving.client` with    * `-max_seq_len=50`    * `-max_seq_len=100` ** failed due to CPU crash **    * `-max_seq_len=200` ** failed due to CPU crash **    We can then compare two models:* uncased_L-12_H-768_A-12 -num_worker=4 ( -max_seq_len=25)* uncased_L-12_H-768_A-12 -num_worker=4 ( -max_seq_len=50)  Increase to 50 tokens<jupyter_code># !bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -max_seq_len=50
%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_tokens50'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_tokens50'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)

# Preparation for testing
# BERT-small, uncased [more tokens]
dir_BSU_train_tokens50 = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_tokens50'
dir_BSU_test_tokens50  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_tokens50'
%%time

hidden_units = [10]
learning_rate = 0.01
bert_model = 'bert_uncased_small_tokens50'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train_tokens50, input_size) # BERT-small, uncased [tokens50]
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
dfTK_50 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['TK_50'])
dfTK_50<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-09:24:54
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_tokens50_DNN_inputNone_maxsteps2000_hu10_lr0.01_dropout0.2/model.ckpt-1563
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-09:24:55
INFO:tensorflow:Saving dict for global step 1563: accuracy = 0.7546, accuracy_baseline = 0.5, auc = 0.83904177, auc_precision_recall = 0.84167016, average_loss = 0.49245283, global_step = 1563, label/mean = 0.5, loss = 61.556602, pre[...]<jupyter_text>Increase to 100 tokens ! FAILED -- CPU Crash<jupyter_code># !bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -max_seq_len=100
%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_tokens100'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_tokens100'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)

# Preparation for testing
# BERT-small, uncased [more tokens]
dir_BSU_train_tokens100 = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_tokens100'
dir_BSU_test_tokens100  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_tokens100'
%%time

hidden_units = [10]
learning_rate = 0.01
bert_model = 'bert_uncased_small_tokens100'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train_tokens100, input_size) # BERT-small, uncased [tokens50]
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
dfTK_100 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['TK_100'])
dfTK_100<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-07-09:24:40
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_tokens50_DNN_inputNone_maxsteps2000_hu10_lr0.01_dropout0.2/model.ckpt-1563
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-07-09:24:41
INFO:tensorflow:Saving dict for global step 1563: accuracy = 0.7546, accuracy_baseline = 0.5, auc = 0.83904177, auc_precision_recall = 0.84167016, average_loss = 0.49245283, global_step = 1563, label/mean = 0.5, loss = 61.556602, pre[...]<jupyter_text>Increase to 200 tokens ! FAILED -- CPU Crash<jupyter_code>%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_tokens200'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_tokens200'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)

# Preparation for testing
# BERT-small, uncased [more tokens]
dir_BSU_train_tokens200 = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_tokens200'
dir_BSU_test_tokens200  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_tokens200'
%%time

hidden_units = [10]
learning_rate = 0.01
bert_model = 'bert_uncased_small_tokens200'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train_tokens200, input_size) # BERT-small, uncased [tokens50]
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
dfTK_200 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['TK_200'])
dfTK_200<jupyter_output><empty_output><jupyter_text>Compare<jupyter_code>pd.concat([df_LR01, dfTK_50], axis=1)<jupyter_output><empty_output><jupyter_text>**TOKEN 25**  accuracy	**0.712600**  accuracy_baseline0.500000  auc	0.794202  auc_precision_recall 0.797760  average_loss 0.544821  global_step	2000.000000  label/mean	0.500000  loss	68.102631  precision	0.727623  prediction/mean	0.5082680  recall 0.679600 **TOKEN 50**  accuracy	**0.754600**  accuracy_baseline	0.500000  auc	0.839042  auc_precision_recall	0.841670  average_loss	0.492453  global_step	1563.000000  label/mean	0.500000  loss	61.556602  precision	0.752479  prediction/mean	0.492644  recall	0.758800   Using 50 tokens significantly increases the accuracy of the model ---      Expiriment: Use end of review---  By default, the BERT model uses the first 25 tokens (beginning of the text). However, we can rebuild the cache with the last 25 words and test performance differences.  We can then compare two sets of two models:* uncased_L-12_H-768_A-12 -num_worker=4 ( -max_seq_len=25), using regular cache  * uncased_L-12_H-768_A-12 -num_worker=4 ( -max_seq_len=25), using cache with only last 25 words      * uncased_L-12_H-768_A-12 -num_worker=4 ( -max_seq_len=50),  using regular cache  * uncased_L-12_H-768_A-12 -num_worker=4 ( -max_seq_len=50), using cache with only last 25 words    last 25 tokens<jupyter_code># rebuild cache (slice last 25 words) to 'test_last25words_tokens25' and 'train_last25words_tokens25'
# bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4
%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_last25words_tokens25'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_last25words_tokens25'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)

# BERT-small, uncased last 25 words
dir_BSU_train_last25 = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_last25words_tokens25'
dir_BSU_test_last25  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_last25words_tokens25'

%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'train_last25words_tokens25'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)

train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train_last25, input_size) # BERT-small, uncased last 25 words
estimator.train(input_fn=train_input_fn, max_steps=2000)
estimator.eval_dir('/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_last25_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2/eval')
# result
dfLAST_25 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LAST_25'])
dfLAST_25<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-12T14:50:28Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_last50_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-12-14:50:29
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.7096, accuracy_baseline = 0.5, auc = 0.7944088, auc_precision_recall = 0.7980648, average_loss = 0.5482111, global_step = 2000, label/mean = 0.5, loss = 68.52638, precisio[...]<jupyter_text>---      last 50 tokens---  In order to compare apples-to apples, we can also rebuild the cache with the last 50 words and test performance differences between first-50 and last-50 tokens<jupyter_code># rebuild cache (slice last 50 words) to 'test_last50words_tokens50' and 'train_last50words_tokens50' using other notebook
# bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -max_seq_len=50
%%time

# set directories
my_dir_train = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/train'
my_dir_test  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/aclImdb/test'
my_dir_train_output = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_last50words_tokens50'
my_dir_test_output  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_last50words_tokens50'

# build cache
input_fn_train = cache_data(my_dir_train, my_dir_train_output, 0, 14)
input_fn_eval = cache_data(my_dir_test, my_dir_test_output, 0, 14)

# BERT-small, uncased last 50 words 
dir_BSU_train_last50 = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/train_last50words_tokens50'
dir_BSU_test_last50  = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/uncased_L-12_H-768_A-12/cache/test_last50words_tokens50'

%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'train_last50words_tokens50'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train_last50, input_size) # BERT-small, uncased last 50 words
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
dfLAST_50 = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['LAST_50'])
dfLAST_50<jupyter_output>INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-04-12T12:27:20Z
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_last50_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2/model.ckpt-2000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-04-12-12:27:20
INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.762, accuracy_baseline = 0.5, auc = 0.8476632, auc_precision_recall = 0.85088396, average_loss = 0.48106036, global_step = 2000, label/mean = 0.5, loss = 60.132545, precis[...]<jupyter_text>ComparisonAccuracies:* **first 25 words** 0.712600* **first 50 words** 0.754600* **last 25 words** 0.709600* **last 50 words** 0.762000Using the last 50 words showed improved performance over using the first 25 or 50 words. However, using only the last 25 words demonstrated a decrease in performnace.  ---      Expiriment: Pooling strategy---  By default, BERT model sets the pooling strategy = REDUCE_MEAN. However, we can modify this in order to see if there are significant performance differences between pooling strategies.   None<jupyter_code># bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -pooling_strategy=NONE
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_uncased_small_pooling_none'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir='/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_none_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2', session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased 
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_PLnone = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLnone'])
df_PLnone
df_PLnone<jupyter_output><empty_output><jupyter_text>Reduce Max<jupyter_code># bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -pooling_strategy=REDUCE_MAX
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_uncased_small_pooling_reducemax'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased 
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
# result
df_PLrm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLrm']<jupyter_output>saving to:

 /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_reducemax_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2 


INFO:tensorflow:Using config: {'_model_dir': '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_reducemax_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x13107d198>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0[...]<jupyter_text>accuracy = 0.6972  accuracy_baseline = 0.5, auc = 0.7893455  auc_precision_recall = 0.7945415  average_loss = 0.5611105  global_step = 2000  label/mean = 0.5  loss = 70.13881  precision = 0.6614276  prediction/mean = 0.5098787  recall = 0.808  Reduce Mean Max<jupyter_code># bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -pooling_strategy=REDUCE_MEAN_MAX
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_uncased_small_pooling_meanmax'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased 
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
df_PLmm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLmm']) # result<jupyter_output>saving to:

 /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_meanmax_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2 


INFO:tensorflow:Using config: {'_model_dir': '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_meanmax_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x131034cc0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_[...]<jupyter_text>accuracy = 0.7116  accuracy_baseline = 0.5  auc = 0.79641646  auc_precision_recall = 0.8001508  average_loss = 0.5378654, global_step = 1563  label/mean = 0.5  loss = 67.23317  precision = 0.7380738  prediction/mean = 0.49952433  recall = 0.656  First Token<jupyter_code># bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -pooling_strategy=FIRST_TOKEN
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_uncased_small_pooling_firstToken'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased 
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
df_PLmm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLmm']) # result<jupyter_output>saving to:

 /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_firstToken_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2 


INFO:tensorflow:Using config: {'_model_dir': '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_firstToken_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x130e38e80>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas':[...]<jupyter_text>accuracy = 0.7168  accuracy_baseline = 0.5  auc = 0.8003941  auc_precision_recall = 0.8031008  average_loss = 0.53677994  global_step = 1563  label/mean = 0.5  loss = 67.09749  precision = 0.71593624  prediction/mean = 0.5240857  recall = 0.7188  Last Token<jupyter_code># bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4 -pooling_strategy=LAST_TOKEN
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_uncased_small_pooling_lastToken'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSU_train, input_size) # BERT-small, uncased 
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
df_PLmm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLmm']) # result<jupyter_output>saving to:

 /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_lastToken_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2 


INFO:tensorflow:Using config: {'_model_dir': '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_uncased_small_pooling_lastToken_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x107cb0710>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0[...]<jupyter_text>accuracy = 0.6988  accuracy_baseline = 0.5  auc = 0.7864142  auc_precision_recall = 0.7919043  average_loss = 0.56336784  global_step = 2000  label/mean = 0.5  loss = 70.42098  precision = 0.66522604  prediction/mean = 0.5048651  recall = 0.8004  Comparison<jupyter_code># pd.concat([df_LR01, df_PLnone, df_PLrm, df_PLmm, df_PLft, df_PLlt], axis=1)<jupyter_output><empty_output><jupyter_text>Accuracy-----  none: 0.7092reduce_max: 0.6972reduce_mean_max: 0.7116first_token = <PASSWORD>last_token = <PASSWORD> ---      Expiriment: case/uncased---    base uncased<jupyter_code># see learning rate experiment
# !bert-serving-start -model_dir ./uncased_L-12_H-768_A-12 -num_worker=4<jupyter_output><empty_output><jupyter_text>accuracy = 0.7126  accuracy_baseline = 0.5  auc = 0.79420185  auc_precision_recall = 0.7977605  average_loss = 0.544821  global_step = 2000  label/mean = 0.5  loss = 68.10263  precision = 0.7276231  prediction/mean = 0.5082678  recall = 0.6796  base cased<jupyter_code># !bert-serving-start -model_dir ./cased_L-12_H-768_A-12 -num_worker=4
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_cased_large'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSC_train, input_size) # BERT-small, cased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSC_train, input_size) # BERT-small, cased 
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
df_PLmm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLmm']) # result<jupyter_output>saving to:

 /Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_cased_large_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2 


INFO:tensorflow:Using config: {'_model_dir': '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_bert_cased_large_DNN_inputNone_maxsteps2000_hu10_lr0.05_dropout0.2', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x107cb09e8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
INFO:tensor[...]<jupyter_text>accuracy = 0.6938  accuracy_baseline = 0.5  auc = 0.74847955  auc_precision_recall = 0.7738552  average_loss = 0.59444726  global_step = 2000  label/mean = 0.5  loss = 74.30591  precision = 0.66274774  prediction/mean = 0.49674615  recall = 0.7892  large uncased ! FAILED -- CPU Crash<jupyter_code># !bert-serving-start -model_dir ./uncased_L-24_H-1024_A-16 -num_worker=4
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_cased_large'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BLU_train, input_size) # BERT-large, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BLU_train, input_size) # BERT-large, uncased 
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
df_PLmm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLmm']) # result<jupyter_output><empty_output><jupyter_text>large cased ! FAILED -- CPU Crash<jupyter_code># !bert-serving-start -model_dir ./cased_L-24_H-1024_A-16 -num_worker=4
%%time

hidden_units = [10]
learning_rate = 0.05
bert_model = 'bert_cased_large'
classifier = 'DNN'
dropout_rate = 0.2
train_max_steps = 2000
input_size = None # all

dir_models = '/Users/jlc/Google Drive/_code/MIDS_W266/BERT_Imdb/trained_models/imdb_{}_{}_input{}_maxsteps{}_hu{}_lr{}_dropout{}'.format(
    bert_model, classifier, input_size, train_max_steps, '_'.join([str(x) for x in hidden_units]), learning_rate, dropout_rate)
print ('\n\nsaving to:\n\n',dir_models, '\n\n')

config = tf.ConfigProto()
run_config = RunConfig(model_dir=dir_models, session_config=config,save_checkpoints_steps=100)
estimator = DNNClassifier(
    hidden_units=hidden_units,
    feature_columns=[tf.feature_column.numeric_column('feature', shape=(768,))],
    n_classes=2,
    config=run_config,
    optimizer=tf.train.AdagradOptimizer(learning_rate=learning_rate),
    dropout=dropout_rate)
train_input_fn, dev_input_fn = get_input_fn(dir_BSC_train, input_size) # BERT-small, uncased
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
train_input_fn, dev_input_fn = get_input_fn(dir_BSC_train, input_size) # BERT-small, uncased 
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
estimator.train(input_fn=train_input_fn, max_steps=train_max_steps)
print('\n\ncheck eval_dir()\n\n',estimator.eval_dir(), '\n\n')
df_PLmm = pd.DataFrame.from_dict(estimator.evaluate(dev_input_fn), orient='index', columns=['PLmm']) # result<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11957059227002281, 'fraction_numerical': 0.056260069228437896, 'mean_word_length': 6.458893515764426, 'pattern_counts': {'":': 0, '<': 110, '<?xml version=': 0, '>': 110, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '30', 'dir': 'jupyter-structured-clean-dedup', 'id': '161652', 'license': 'MIT', 'n_tokens_mistral': 25761, 'n_tokens_neox': 23277, 'n_words': 3944, 'path': '_notebooks/BERT Imdb experiments.ipynb', 'repo_name': 'jcalvillo/BERT_Imdb', 'size': '62683'}