<jupyter_start><jupyter_text>Data loader experiments<jupyter_code>#export
from local.imports import *
from local.test import *
from local.core import *
from local.data import *
from local.notebook.showdoc import show_doc<jupyter_output><empty_output><jupyter_text>patch \_MapDatasetFetcher<jupyter_code>@patch
def fetch(self:torch.utils.data._utils.fetch._MapDatasetFetcher, idxs):
    ds = self.dataset
    if self.auto_collation:
        if hasattr(self.collate_fn,"get_batch"): return self.collate_fn.get_batch(ds, idxs)
        data = [ds[idx] for idx in idxs]
    else: data = ds[idxs]
    return self.collate_fn(data)
class CustCollate:
    def __init__(self, cf=default_collate): self.cf=cf
    def __call__(self, *args, **kwargs): self.cf(*args, **kwargs)
    def get_batch(self, ds, idxs): return self([ds[idx] for idx in idxs])
ds = string.ascii_lowercase
dl = DataLoader(ds, batch_size=4, num_workers=0)#, collate_fn=CustCollate())
[''.join(o) for o in dl]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12537612838515547, 'fraction_numerical': 0.0020060180541624875, 'mean_word_length': 5.565789473684211, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '171996', 'license': 'Apache-2.0', 'n_tokens_mistral': 361, 'n_tokens_neox': 364, 'n_words': 77, 'path': 'dev/_04a_dloader.ipynb', 'repo_name': 'nareshr8/fastai_dev', 'size': '997'}