<jupyter_start><jupyter_text>--- Initialise Tokeniser & Model<jupyter_code>tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
model = DistilBertModel.from_pretrained('distilbert-base-uncased')

model.eval()<jupyter_output>I0224 12:16:56.094146 140035151705920 configuration_utils.py:254] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json from cache at /home/valentin/.cache/torch/transformers/a41e817d5c0743e29e86ff85edc8c257e61bc8d88e4271bb1b243b6e7614c633.587f67ec28c540f4294c9c2ac7dcf7841ff371aeb12cdeb6a17f69da39ad9452
I0224 12:16:56.096509 140035151705920 configuration_utils.py:290] Model config DistilBertConfig {
  "activation": "gelu",
  "architectures": [
    "DistilBertForMaskedLM"
  ],
  "attention_dropout": 0.1,
  "bos_token_id": 0,
  "dim": 768,
  "do_sample": false,
  "dropout": 0.1,
  "eos_token_ids": 0,
  "finetuning_task": null,
  "hidden_dim": 3072,
  "id2label": {
    "0": "LABEL_0",
    "1": "LABEL_1"
  },
  "initializer_range": 0.02,
  "is_decoder": false,
  "label2id": {
    "LABEL_0": 0,
    "LABEL_1": 1
  },
  "length_penalty": 1.0,
  "max_length": 20,
  "max_position_embeddings": 512,
  "model_type": "distilbert",
  "n_he[...]<jupyter_text>Defs convert email text to model input and then model output<jupyter_code>def email_to_ids(email_body):
    return torch.tensor(tokenizer.encode(email_body, add_special_tokens=True))

def email_to_vec(email_body_ids, to_id_first=True):
    if to_id_first:
        input_ids = email_to_ids(email_body_ids)
    else:
        input_ids = email_body_ids
        
    input_ids = input_ids.unsqueeze(0)
    outputs = model(input_ids)
    return outputs
    
#     last_hidden_states = outputs[0][0]
#     return last_hidden_states.mean(0)  # mean vs sum<jupyter_output><empty_output><jupyter_text>--- Load emails and apply tokeniser and conversion to model input<jupyter_code>with open("../../w3c-emails/emails.pkl", "rb") as handle:
    emails = pickle.load(handle)
texts = [e.body_raw for e in emails]
tok_ids = [email_to_ids(t) for t in tqdm(texts)]
with open("emails_token_ids.pkl", "wb") as handle:
    pickle.dump(tok_ids, handle)
exit()<jupyter_output><empty_output><jupyter_text>--- Cut up emails longer than 512 tokens & group into batches(start from here by loading converted emails from pkl)<jupyter_code>with open("emails_token_ids.pkl", "rb") as handle:
    ids = pickle.load(handle)
cut_up = [id_tens.split(511) for id_tens in ids]
with torch.no_grad():    
    vecs = [[email_to_vec(v, to_id_first=False) for v in tt] for tt in tqdm(cut_up[:5000])]
    
with open("vectors2.pkl", "wb") as handle:
    pickle.dump(vecs, handle)
    
exit()
with open("vectors.pkl", "rb") as handle:
    vecs_loaded = pickle.load(handle)
len(vecs_loaded), len(vecs_loaded[0]), vecs_loaded[0][0].shape
lens = [len(t) for t in texts if len(t) < ]

plt.hist(lens, bins=30)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13164128595600677, 'fraction_numerical': 0.06802030456852792, 'mean_word_length': 5.094845360824742, 'pattern_counts': {'":': 22, '<': 17, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '433159', 'license': 'Apache-2.0', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1159, 'n_words': 213, 'path': 'analytics/embeddings_for_the_people/raw_BERT/BERT_on_emails.ipynb', 'repo_name': 'INDElab/conversationkg', 'size': '2955'}