<jupyter_start><jupyter_text>Understanding BERT Sentence VectorizerFollowing what's described here:https://github.com/google-research/bertusing-bert-to-extract-fixed-feature-vectors-like-elmo  Input Text<jupyter_code>with open('./input_fra.txt') as fin:
    inp = fin.read()
    print(inp)
with open('./output_fra.jsonl') as fin:
    output_jsonl = fin.read()
import json
json_lines = output_jsonl.split('\n')
line = json.loads(json_lines[0])
line.keys()
line['features'][0].keys()
for feature in line['features']:
  print(feature['token'])
feature.keys()
for layer in feature['layers']:
  print(layer['index'])
len(layer['values'])
for feature in line['features']:
  print(feature['token'])
  for layer in feature['layers']:
    print('  ', layer['index'], len(layer['values']))

import matplotlib.pyplot as plt
plt.plot(layer['values'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1716589861751152, 'fraction_numerical': 0.002304147465437788, 'mean_word_length': 6.241666666666666, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '151605', 'license': 'MIT', 'n_tokens_mistral': 296, 'n_tokens_neox': 291, 'n_words': 58, 'path': 'notebooks/Understanding_BERT_Sentence_Vectorizer.ipynb', 'repo_name': 'BioMedBERT/biomedbert', 'size': '868'}