<jupyter_start><jupyter_text>Use Kaggle's My Account page to down load a kaggle.json file and re-upload it here.<jupyter_code>!mkdir -p /root/.kaggle
token_file = "/root/.kaggle/kaggle.json"
uploaded = files.upload()
with open(token_file, "wb") as f:
  f.write(uploaded["kaggle.json"])
  os.chmod(token_file, 600)
import kaggle<jupyter_output><empty_output><jupyter_text>Download the test set and extract the labeled portion<jupyter_code>kaggle.api.competition_download_file('jigsaw-toxic-comment-classification-challenge', 'test.csv')
kaggle.api.competition_download_file('jigsaw-toxic-comment-classification-challenge', 'test_labels.csv')
test_labels = pd.read_csv('test_labels.csv.zip', index_col='id')
testset = test_labels.loc[test_labels['toxic'] != -1].join(
  pd.read_csv('test.csv.zip', index_col='id'))<jupyter_output><empty_output><jupyter_text>Load the pre-trained toxicity model from Google Cloud Storage<jupyter_code>auth.authenticate_user()
!mkdir -p tfjs_model
!gsutil -m cp -R gs://conversationai-public/public_models/tfjs/v1/* tfjs_model
predict_fn = tf.contrib.predictor.from_saved_model(
  'tfjs_model', signature_def_key='predict')<jupyter_output>WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/predictor/saved_model_predictor.py:153: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be [...]<jupyter_text>Load sentence piece model and preprocess test data<jupyter_code>sp = sentencepiece.SentencePieceProcessor()
sp.Load('tfjs_model/assets/universal_encoder_8k_spm.model')<jupyter_output><empty_output><jupyter_text>Score the sentences with toxicity model<jupyter_code>def progress(value, max=100):
    return HTML("""
        <progress
            value='{value}'
            max='{max}',
            style='width: 100%'
        >
            {value}
        </progress>
    """.format(value=value, max=max))
tox_scores = []
nrows = testset.shape[0]
out = display(progress(0, nrows), display_id=True)
for offset in range(0, nrows):
  out.update(progress(offset, nrows))
  values = sp.EncodeAsIds(testset['comment_text'][offset])
  tox_scores.append(predict_fn({
      'values': values,
      'indices': [(0, i) for i in range(len(values))],
      'dense_shape': [1, len(values)]})['toxicity/probabilities'][0,1])
!gsutil cp gs://conversationai-public/public_models/tfjs/perspectiveapi.csv .
perspective_api=pd.read_csv('perspectiveapi.csv')
top_kernel = kaggle.api.kernels_output(kernel='tunguz/superblend', path='.')
top_kernel_scores = testset = test_labels.loc[test_labels['toxic'] != -1].join(
  pd.read_csv('superblend.csv', index_col='id'), rsuffix='_predicted')
plt.figure()

fpr, tpr, _ = metrics.roc_curve(testset['toxic'], tox_scores)
plt.plot(fpr, tpr, label='Tensorflow JS model')

fpr, tpr, _ = metrics.roc_curve(testset['toxic'],
                                perspective_api['PerspectiveAPI'].values)
plt.plot(fpr, tpr, label='Perspective API')


fpr, tpr, _ = metrics.roc_curve(top_kernel_scores['toxic'],
                                top_kernel_scores['toxic_predicted'])
plt.plot(fpr, tpr, label='Top scoring Kaggle kernel')

plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.legend(loc='lower right')
plt.ylim(0.75, 1.0)
plt.xlim(0.0, 0.25)
plt.title('Performance on Kaggle Toxic Comments Challenge Test Set')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1333492937514963, 'fraction_numerical': 0.01340675125688293, 'mean_word_length': 5.33030303030303, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '481440', 'license': 'Apache-2.0', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1396, 'n_words': 276, 'path': 'experiments/tf_trainer/tf_hub_tfjs/notebook/EvaluatingClassifier.ipynb', 'repo_name': 'Newander/conversationai-models', 'size': '4177'}