<jupyter_start><jupyter_text>Word, token lengths<jupyter_code>%%time
col = "qc_length"
df[col] = df["question"].str.len() + df["context"].str.len() 
df[col] = df[col].astype(np.int32)
%%time
col = "a_length"
df[col] = df["answer_text"].str.len()
df[col] = df[col].astype(np.int32)
def word_length(cols: Iterable) -> Callable:
    def f(row) -> int:
        res = 0
        for col in cols:
            res += len(row[col].split())
        return res
    
    return f
%%time
col = "qc_word_length"
df[col] = df.progress_apply(word_length(["question", "context"]), axis=1)
df[col] = df[col].astype(np.int32)
%%time
col = "a_word_length"
df[col] = df.progress_apply(word_length(["answer_text"]), axis=1)
df[col] = df[col].astype(np.int32)
#pretrained_dir = "../pretrained/google/electra-small-discriminator"
#tokenizer = AutoTokenizer.from_pretrained(pretrained_dir, model_max_length=512)
#print(f"{repr(tokenizer)}\n{tokenizer.model_input_names}")
#pretrained_dir = "../pretrained/albert-base-v2"
#sp_tokenizer = AutoTokenizer.from_pretrained(pretrained_dir, model_max_length=512)
#print(f"{repr(sp_tokenizer)}\n{sp_tokenizer.model_input_names}")
#pretrained_dir = "../pretrained/distilroberta-base"
#bpe_tokenizer = AutoTokenizer.from_pretrained(pretrained_dir, model_max_length=512)
#print(f"{repr(bpe_tokenizer)}\n{bpe_tokenizer.model_input_names}")
#%%time
#x = tokenizer(questions, contexts)
#print(f"{repr(x.keys())}\nlen={len(x['input_ids'])}")
#col = "qc_wp_length"
#df[col] = [len(v) for v in x["input_ids"]]
#df[col] = df[col].astype(np.int16)
#%%time
#x = sp_tokenizer(questions, contexts)
#print(f"{repr(x.keys())}\nlen={len(x['input_ids'])}")
#col = "qc_sp_length"
#df[col] = [len(v) for v in x["input_ids"]]
#df[col] = df[col].astype(np.int16)
#%%time
#x = bpe_tokenizer(questions, contexts)
#print(f"{repr(x.keys())}\nlen={len(x['input_ids'])}")
#col = "qc_bpe_length"
#df[col] = [len(v) for v in x["input_ids"]]
#df[col] = df[col].astype(np.int16)
#cols = ["qc_length", "a_length", "qc_word_length", "a_word_length", 
#        "qc_wp_length", "qc_sp_length", "qc_bpe_length"]
#df[cols].describe(percentiles=percentiles)
df.info()
%%time
df.to_parquet("output/train.parquet", index=False)<jupyter_output>Wall time: 323 ms
{'fraction_non_alphanumeric': 0.20313199105145413, 'fraction_numerical': 0.013422818791946308, 'mean_word_length': 6.710344827586207, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '23765', 'license': 'MIT', 'n_tokens_mistral': 961, 'n_tokens_neox': 928, 'n_words': 132, 'path': 'squad20/preprocess.ipynb', 'repo_name': 'seahrh/question-answering', 'size': '2235'}