<jupyter_start><jupyter_text>Pytorch datasets<jupyter_code>#!pip -q install Datasets
#!pip -q install transformers
import os
import gc
import copy
import numpy as np
import transformers
from transformers import BertTokenizerFast,BertForSequenceClassification, BertModel, BertConfig

from datasets import Dataset, load_metric
import datasets

import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
tokenizer = BertTokenizerFast.from_pretrained('bert-base-chinese')
config = {
    'max_len': 512,
}
def preprocess_function(examples):
    # prompt[CLS]instance[SEP]label[SEP]
    
    LEN = len(examples["instance"])
    max_length = config['max_len'] 
    # Repeat each first sentence four times to go with the four possibilities of second sentences.
    contexts = [[context] * numchoice for context, numchoice in zip(examples["instance"], examples['numchoices'])]
    candidates = [[f'{cand}' for cand in 
                 tpclass.split(',')] for word, tpclass in zip(examples['word'], examples['zh_type_class'])]
    
    # Flatten everything
    first_sentences = sum(contexts, [])
    second_sentences = sum(candidates, [])
    # Tokenize
    tokenized_examples = tokenizer(first_sentences, 
                                   second_sentences,
                                   padding = False, 
                                   truncation=True,
                                   return_token_type_ids=True,)
                                   #add_special_tokens = True) # default to True
    # Un-flatten
    # match the correct number of instances back 
    # return {k: [v[i:i+numchoice] for i in range(0, len(v), numchoice)] for k, v in tokenized_examples.items()}
    numchoices = examples['numchoices']
    tok_examples = {}
    
    for k, v in tokenized_examples.items():
        print('flattened instances #', len(v))
        print(k)
        unflattend_v = []
        currstart, currend = 0, 0
        idx = 0 
        while currstart < len(v):
            numchoice  = numchoices[idx]
            currend += numchoice
            instances = v[currstart:currstart+currend]
            unflattend_v.append(instances)
            # print('!', numchoice, currstart, currend)
            currstart = currend
            idx += 1
        tok_examples[k] = unflattend_v 
    fix_len = len(tok_examples['input_ids'])
    print('length:', fix_len)
    if not all(len(tok_examples[k]) == fix_len for k in tok_examples):
         raise ValueError('not all lists have same length!')
    return tok_examples
TESTSIZE = 0.2
dataset = Dataset.from_pandas(df)
trainset, testset = dataset.train_test_split(test_size=TESTSIZE).values()
final_dataset = datasets.DatasetDict({"train":trainset,"test":testset})
idx = 111
examples = final_dataset["train"][:300]
features = preprocess_function(examples)
nc = df["numchoices"].iloc[idx]
print(nc)
[tokenizer.decode(features["input_ids"][idx][i]) for i in range(nc)]
## 為啥tokenizer裡沒有return其他columns?還是說是dataset.map()才會有<jupyter_output>flattened instances # 617
input_ids
flattened instances # 617
token_type_ids
flattened instances # 617
attention_mask
length: 300
2<jupyter_text>Encoding & saving<jupyter_code>encoded_dataset = final_dataset.map(preprocess_function, batched = True)
# PT for prompt-tuning
fpath = f"{datadir}/PT2_allchoice_encoded_dataset"
encoded_dataset.save_to_disk(fpath)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10161054172767203, 'fraction_numerical': 0.008784773060029283, 'mean_word_length': 3.357142857142857, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '627344', 'license': 'MIT', 'n_tokens_mistral': 1097, 'n_tokens_neox': 1051, 'n_words': 245, 'path': 'notebooks/11.0_preparing_allchoice_dataset.ipynb', 'repo_name': 'Nana2929/Miscellaneous', 'size': '3415'}