how to convert a dict generator into a huggingface dataset. 
{'login': 'StephennFernandes', 'id': 32235549, 'node_id': 'MDQ6VXNlcjMyMjM1NTQ5', 'avatar_url': 'https://avatars.githubusercontent.com/u/32235549?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/StephennFernandes', 'html_url': 'https://github.com/StephennFernandes', 'followers_url': 'https://api.github.com/users/StephennFernandes/followers', 'following_url': 'https://api.github.com/users/StephennFernandes/following{/other_user}', 'gists_url': 'https://api.github.com/users/StephennFernandes/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/StephennFernandes/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/StephennFernandes/subscriptions', 'organizations_url': 'https://api.github.com/users/StephennFernandes/orgs', 'repos_url': 'https://api.github.com/users/StephennFernandes/repos', 'events_url': 'https://api.github.com/users/StephennFernandes/events{/privacy}', 'received_events_url': 'https://api.github.com/users/StephennFernandes/received_events', 'type': 'User', 'site_admin': False}
[{'id': 1935892912, 'node_id': 'MDU6TGFiZWwxOTM1ODkyOTEy', 'url': 'https://api.github.com/repos/huggingface/datasets/labels/question', 'name': 'question', 'color': 'd876e3', 'default': True, 'description': 'Further information is requested'}]
{'login': 'mariosasko', 'id': 47462742, 'node_id': 'MDQ6VXNlcjQ3NDYyNzQy', 'avatar_url': 'https://avatars.githubusercontent.com/u/47462742?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/mariosasko', 'html_url': 'https://github.com/mariosasko', 'followers_url': 'https://api.github.com/users/mariosasko/followers', 'following_url': 'https://api.github.com/users/mariosasko/following{/other_user}', 'gists_url': 'https://api.github.com/users/mariosasko/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/mariosasko/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/mariosasko/subscriptions', 'organizations_url': 'https://api.github.com/users/mariosasko/orgs', 'repos_url': 'https://api.github.com/users/mariosasko/repos', 'events_url': 'https://api.github.com/users/mariosasko/events{/privacy}', 'received_events_url': 'https://api.github.com/users/mariosasko/received_events', 'type': 'User', 'site_admin': False}
[{'login': 'mariosasko', 'id': 47462742, 'node_id': 'MDQ6VXNlcjQ3NDYyNzQy', 'avatar_url': 'https://avatars.githubusercontent.com/u/47462742?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/mariosasko', 'html_url': 'https://github.com/mariosasko', 'followers_url': 'https://api.github.com/users/mariosasko/followers', 'following_url': 'https://api.github.com/users/mariosasko/following{/other_user}', 'gists_url': 'https://api.github.com/users/mariosasko/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/mariosasko/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/mariosasko/subscriptions', 'organizations_url': 'https://api.github.com/users/mariosasko/orgs', 'repos_url': 'https://api.github.com/users/mariosasko/repos', 'events_url': 'https://api.github.com/users/mariosasko/events{/privacy}', 'received_events_url': 'https://api.github.com/users/mariosasko/received_events', 'type': 'User', 'site_admin': False}]
['@albertvillanova @lhoestq , could you please help me on this issue. '
 'Hi ! As mentioned on the [forum](https://discuss.huggingface.co/t/how-to-wrap-a-generator-with-hf-dataset/18464), the simplest for now would be to define a [dataset script](https://huggingface.co/docs/datasets/dataset_script) which can contain your generator. But we can also explore adding something like `ds = Dataset.from_iterable(seqio_dataset)`'
 '@lhoestq , hey i did as you instructed, but sadly i cannot get pass through the download_manager, as i dont have anything to download. i was skipping the ` def _split_generators(self, dl_manager):` function. but i cannot get around it. I get a `NotImplementedError: `\r\n\r\nthe following is my code for the same: \r\n\r\n\r\n\r\n```\r\nimport datasets \r\nimport functools\r\nimport glob \r\nfrom datasets import load_from_disk\r\nimport seqio\r\nimport tensorflow as tf\r\nimport t5.data\r\nfrom datasets import load_dataset\r\nfrom t5.data import postprocessors\r\nfrom t5.data import preprocessors\r\nfrom t5.evaluation import metrics\r\nfrom seqio import FunctionDataSource, utils\r\n\r\nTaskRegistry = seqio.TaskRegistry\r\n\r\ndata_path = glob.glob("/home/stephen/Desktop/MEGA_CORPUS/COMBINED_CORPUS/*", recursive=False)\r\n\r\n\r\ndef gen_dataset(split, shuffle=False, seed=None, column="text", dataset_path=None):\r\n    dataset = load_from_disk(dataset_path)\r\n    if shuffle:\r\n        if seed:\r\n            dataset = dataset.shuffle(seed=seed)\r\n        else:\r\n            dataset = dataset.shuffle()\r\n    while True:\r\n        for item in dataset[str(split)]:\r\n            yield item[column]\r\n\r\n\r\ndef dataset_fn(split, shuffle_files, seed=None, dataset_path=None):\r\n    return tf.data.Dataset.from_generator(\r\n        functools.partial(gen_dataset, split, shuffle_files, seed, dataset_path=dataset_path),\r\n        output_signature=tf.TensorSpec(shape=(), dtype=tf.string, name=dataset_path)\r\n    )\r\n\r\n@utils.map_over_dataset\r\ndef target_to_key(x, key_map, target_key):\r\n    """Assign the value from the dataset to target_key in key_map"""\r\n    return {**key_map, target_key: x}\r\n\r\n\r\n_CITATION = "Not ready yet"\r\n_DESCRIPTION = "a custom seqio based mixed samples on a given temperature value, that again returns a dataset in HF dataset format well samples on the Mixture temperature"\r\n_HOMEPAGE = "ldcil.org"\r\n\r\nclass CustomSeqio(datasets.GeneratorBasedBuilder):\r\n\r\n    def _info(self):\r\n        return datasets.DatasetInfo(\r\n            description=_DESCRIPTION,\r\n            features=datasets.Features(\r\n                {\r\n                    "text": datasets.Value("string"),\r\n                }\r\n            ),\r\n            homepage="https://ldcil.org",\r\n            citation=_CITATION,)\r\n\r\ndef generate_examples(self):\r\n    seqio_train_list = []\r\n    for lang in data_path:\r\n        dataset_name = lang.split("/")[-1]\r\n        dataset_shapes = None \r\n\r\n        TaskRegistry.add(\r\n        str(dataset_name),\r\n        source=seqio.FunctionDataSource(\r\n            dataset_fn=functools.partial(dataset_fn, dataset_path=lang),\r\n            splits=("train", "test"),\r\n            caching_permitted=False,\r\n            num_input_examples=dataset_shapes,\r\n        ),\r\n        preprocessors=[\r\n        functools.partial(\r\n        target_to_key, key_map={\r\n        "targets": None,\r\n        }, target_key="targets")],\r\n            output_features={"targets": seqio.Feature(vocabulary=seqio.PassThroughVocabulary, add_eos=False, dtype=tf.string, rank=0)},\r\n            metric_fns=[]\r\n        )\r\n\r\n        seqio_train_dataset = seqio.get_mixture_or_task(dataset_name).get_dataset(\r\n        sequence_length=None,\r\n        split="train",\r\n        shuffle=True,\r\n        num_epochs=1,\r\n        shard_info=seqio.ShardInfo(index=0, num_shards=10),\r\n        use_cached=False,\r\n        seed=42)\r\n        seqio_train_list.append(seqio_train_dataset)\r\n    \r\n    lang_name_list = []\r\n    for lang in data_path:\r\n        lang_name = lang.split("/")[-1]\r\n        lang_name_list.append(lang_name)\r\n\r\n    seqio_mixture = seqio.MixtureRegistry.add(\r\n        "seqio_mixture",\r\n        lang_name_list,\r\n        default_rate=0.7)\r\n    \r\n    seqio_mixture_dataset = seqio.get_mixture_or_task("seqio_mixture").get_dataset(\r\n    sequence_length=None,\r\n    split="train",\r\n    shuffle=True,\r\n    num_epochs=1,\r\n    shard_info=seqio.ShardInfo(index=0, num_shards=10),\r\n    use_cached=False,\r\n    seed=42)\r\n\r\n    for id, ex in  enumerate(seqio_mixture_dataset):\r\n        yield id, {"text": ex["targets"].numpy().decode()}\r\n```\r\n\r\nand i load it by:\r\n\r\n`seqio_mixture = load_dataset("seqio_loader")`'
 '@lhoestq , just to make things clear ... \r\n\r\nthe following is my original code, thats not in the HF dataset loading script: \r\n\r\n```\r\nimport functools\r\nimport seqio\r\nimport tensorflow as tf\r\nimport t5.data\r\nfrom datasets import load_from_disk\r\nfrom t5.data import postprocessors\r\nfrom t5.data import preprocessors\r\nfrom t5.evaluation import metrics\r\nfrom seqio import FunctionDataSource, utils\r\nimport glob \r\n\r\nTaskRegistry = seqio.TaskRegistry\r\n\r\n\r\n\r\ndef gen_dataset(split, shuffle=False, seed=None, column="text", dataset_path=None):\r\n    dataset = load_from_disk(dataset_path)\r\n    if shuffle:\r\n        if seed:\r\n            dataset = dataset.shuffle(seed=seed)\r\n        else:\r\n            dataset = dataset.shuffle()\r\n    while True:\r\n        for item in dataset[str(split)]:\r\n            yield item[column]\r\n\r\n\r\ndef dataset_fn(split, shuffle_files, seed=None, dataset_path=None):\r\n    return tf.data.Dataset.from_generator(\r\n        functools.partial(gen_dataset, split, shuffle_files, seed, dataset_path=dataset_path),\r\n        output_signature=tf.TensorSpec(shape=(), dtype=tf.string, name=dataset_path)\r\n    )\r\n\r\n\r\n@utils.map_over_dataset\r\ndef target_to_key(x, key_map, target_key):\r\n    """Assign the value from the dataset to target_key in key_map"""\r\n    return {**key_map, target_key: x}\r\n\r\ndata_path = glob.glob("/home/stephen/Desktop/MEGA_CORPUS/COMBINED_CORPUS/*", recursive=False)\r\n\r\nseqio_train_list = []\r\n\r\nfor lang in data_path:\r\n    dataset_name = lang.split("/")[-1]\r\n    dataset_shapes = None \r\n\r\n    TaskRegistry.add(\r\n    str(dataset_name),\r\n    source=seqio.FunctionDataSource(\r\n        dataset_fn=functools.partial(dataset_fn, dataset_path=lang),\r\n        splits=("train", "test"),\r\n        caching_permitted=False,\r\n        num_input_examples=dataset_shapes,\r\n    ),\r\n    preprocessors=[\r\n    functools.partial(\r\n    target_to_key, key_map={\r\n    "targets": None,\r\n    }, target_key="targets")],\r\n        output_features={"targets": seqio.Feature(vocabulary=seqio.PassThroughVocabulary, add_eos=False, dtype=tf.string, rank=0)},\r\n        metric_fns=[]\r\n    )\r\n\r\n    seqio_train_dataset = seqio.get_mixture_or_task(dataset_name).get_dataset(\r\n    sequence_length=None,\r\n    split="train",\r\n    shuffle=True,\r\n    num_epochs=1,\r\n    shard_info=seqio.ShardInfo(index=0, num_shards=10),\r\n    use_cached=False,\r\n    seed=42)\r\n    seqio_train_list.append(seqio_train_dataset)\r\n\r\nlang_name_list = []\r\nfor lang in data_path:\r\n    lang_name = lang.split("/")[-1]\r\n    lang_name_list.append(lang_name)\r\n\r\nseqio_mixture = seqio.MixtureRegistry.add(\r\n  "seqio_mixture",\r\n  lang_name_list,\r\n  default_rate=0.7\r\n)\r\n\r\nseqio_mixture_dataset = seqio.get_mixture_or_task("seqio_mixture").get_dataset(\r\n    sequence_length=None,\r\n    split="train",\r\n    shuffle=True,\r\n    num_epochs=1,\r\n    shard_info=seqio.ShardInfo(index=0, num_shards=10),\r\n    use_cached=False,\r\n    seed=42)\r\n\r\nfor _, ex in zip(range(15), seqio_mixture_dataset):\r\n    print(ex["targets"].numpy().decode())\r\n```\r\n\r\nwhere the seqio_mixture_dataset is the generator that i wanted to be wrapped in HF dataset. \r\n\r\nalso additionally, could you please tell me how do i set the `default_rate=0.7` args where `seqio_mixture` is defined to be made as a custom option in the HF load_dataset() method,\r\n\r\nmaybe like this: \r\n`seqio_mixture_dataset = datasets.load_dataset("seqio_loader",temperature=0.5)`'
 'I like the idea of having `Dataset.from_iterable(iterable)` in the API. The only problem is that we also want to make this part cachable, which is tricky if `iterable` is a generator. \r\n\r\nSome resources on this issue:\r\n* https://github.com/uqfoundation/dill/issues/311\r\n* https://stackoverflow.com/questions/7180212/why-cant-generators-be-pickled\r\n* https://github.com/tonyroberts/generator_tools - python package for pickling generators; pickles bytecode, so it creates version-specific dumps'
 'For the caching maybe we can have `Dataset.from_generator` as TF and pickle+hash the generator function (not the generator object itself) ?\r\n\r\nAnd then keep `Dataset.from_iterable` fo pickable objects like lists'
 '@lhoestq, @mariosasko do you too have any examples where the dataset is a generator and needs to be wrapped into hf dataset ? '
 '@lhoestq, following to my previous question ... what possibly could be done in this [link1](https://github.com/huggingface/datasets/issues/4417#issuecomment-1146627404) [link2](https://github.com/huggingface/datasets/issues/4417#issuecomment-1146627593) case? do you have any ideas? '
 '@lhoestq +1 for the `Dataset.from_generator` idea.\r\n\r\nHaving thought about it, let\'s avoid adding `Dataset.from_iterable` to the API since dictionaries are technically iteralbles ("iterable" is a broad term in Python), and we already provide `Dataset.from_dict`. And for lists maybe we can add `Dataset.from_list` similar to `pa.Table.from_pylist`. WDYT?\r\n'
 'Hi @StephennFernandes!\r\n\r\nTo fix the issues in the copied code, rename `generate_examples` to` _generate_examples` and add one level of indentation as this is a method of `GeneratorBasedBuilder` and define `_split_generators` as follows (again as a method of `GeneratorBasedBuilder):\r\n```python\r\n def _split_generators(self, dl_manager):\r\n        return [\r\n            datasets.SplitGenerator(\r\n                name=datasets.Split.TRAIN,\r\n                gen_kwargs={},\r\n            ),\r\n        ]\r\n```\r\n\r\nAnd if you are feeling extra adventurous, you can try to use ArrowWriter to directly create a cache file:\r\n```python\r\nfrom datasets import Dataset\r\nfrom datasets.arrow_writer import ArrowWriter\r\n\r\nwriter = ArrowWriter(path="path/to/cache_file.arrow", writer_batch_size=1000)\r\n\r\nwith writer:\r\n    for ex in generator:\r\n        writer.write(ex) \r\n    writer.finalize()\r\n\r\ndset = Dataset.from_file("path/to/cache_file.arrow")\r\n```\r\n\r\n'
 'I have a problem which I think is very similar: I would like to "stream" data to a HF Array (memory-mapped) Dataset, where the final size of the dataset is unknown, but could be much larger than what fits into memory.\r\nWhat I want to end up with is an Array Dataset which I can open using `Dataset.load_from_disk(dataset_path="somename")` and use e.g. as the training set. \r\n\r\nFor this I would have thought there should be an API which allows me to open/create the dataset (and define the features etc), then write examples to the dataset, but I could not find a way to do this. \r\n\r\nI tried doing this and it looks like it works, but it feels very hacky and I am not sure if this might fail to update some of the fields in the json files which may turn out to be important:\r\n```\r\nfrom datasets import Dataset, Features, ClassLabel, Sequence, Value\r\nfrom datasets.arrow_writer import ArrowWriter                                                                                                                                                   \r\n# 1) define the features\r\nfeatures = Features(dict(\r\n    id=Value(dtype="string"),\r\n    tokens=Sequence(feature=Value(dtype="string")),\r\n    ner_tags=Sequence(feature=ClassLabel(names=[\'O\', \'B-corporation\', \'I-corporation\', \'B-creative-work\', \'I-creative-work\', \'B-group\', \'I-group\', \'B-location\', \'I-location\', \'B-person\', \'I-person\', \'B-product\', \'I-product\'])),\r\n))\r\n# 2) create empty dataset for examples with these features and store to disk\r\nempty = dict(\r\n    id = [],\r\n    tokens = [],\r\n    ner_tags = [],\r\n)\r\nds = Dataset.from_dict(empty, features=features)\r\nds.save_to_disk(dataset_path="debug_ds1")\r\n\r\n# 3) directly write all the examples to the arrow dataset \r\nwith ArrowWriter(path="debug_ds1/dataset.arrow") as writer: \r\n    writer.write(dict(id=0, tokens=["a", "b"], ner_tags=[0, 0])) \r\n    writer.write(dict(id=1, tokens=["x", "y"], ner_tags=[1, 0])) \r\n    writer.finalize() \r\n \r\nds2 = Dataset.load_from_disk(dataset_path="debug_ds1")\r\nlen(ds2)\r\n```\r\nIs there a cleaner/proper way to do this?\r\n\r\nI like the sound of `Dataset.from_iterable` or `Dataset.from_generator` (should not from iterable be able to handle from generator too as all generators are iterables?) but how would I define the features for me examples there? '
 "Hi @johann-petrak! You can pass the features directly to ArrowWriter's initializer like so `ArrowWriter(..., features=features)`.\r\n\r\nAnd the reason why I prefer `Dataset.from_generator` over `Dataset.from_iterable` is mentioned in one of my previous comments."
 '@mariosasko so at the moment we still have to create a fake `Dataset` first and then use `ArrowWriter` to write an actual dataset? I\'m using the latest version of `datasets` on pypi but my final file is always empty. Is there anything wrong with the code below?\r\n\r\n```python\r\n    total = 0\r\n    with ArrowWriter(path=str(final_data_path), features=features) as writer:\r\n        for batch in loader:\r\n            for traj in batch:\r\n                for generator in question_generators:\r\n                    for xi in generator(traj):\r\n                        # print(f"Question: {xi.question}, answer: {xi.answer}")\r\n                        total += 1\r\n                        writer.write(\r\n                            {\r\n                                "id": f"qa_{total}",\r\n                                "question": xi.question,\r\n                                "answer": xi.answer,\r\n                            }\r\n                        )\r\n        writer.finalize()\r\n    print(f"Total #questions = {total}") # this prints 402\r\n```'
 'This works for me if I then (actually I also close the writer: `writer.close()`)  open the Arrow file as a dataset using `ds=Dataset.from_file(final_data_path)` then `ds.save_to_disk(somedir)`. The Dataset created that way contains the expected examples.'
 "Oh thanks. That did the trick I believe. Shouldn't ArrowWriter have a context manager that does these operations?"
 'You can just use `Dataset.from_file` to get your dataset, no need to do an extra `save_to_disk` somewhere else ;)'
 'I was thinking that `save_to_disk` is necessary when one wants to re-use that dataset as a proper HF dataset later, no?\r\nAt least what I wanted to achieve is create a dataset that can be opened like any other local or remote dataset. '
 "`save_to_disk`/`load_from_disk` is indeed more general, e.g. it supports datasets that consist in several files, and saves some extra info in a dataset_info.json file (description, citation, split sizes, etc.)\r\n\r\nIf you have one single file it's fine to simply do `.from_file()`"]
### Link

_No response_

### Description

Hey there, I have used seqio to get a well distributed mixture of samples from multiple dataset. However the resultant output from seqio is a python generator dict, which I cannot produce back into huggingface dataset.

The generator contains all the samples needed for training the model but I cannot convert it into a huggingface dataset.

The code looks like this:


```
for ex in seqio_data:
print(ex[“text”])
```

I need to convert the seqio_data (generator) into huggingface dataset.


the complete seqio code goes here:
```
import functools

import seqio
import tensorflow as tf
import t5.data
from datasets import load_dataset
from t5.data import postprocessors
from t5.data import preprocessors
from t5.evaluation import metrics
from seqio import FunctionDataSource, utils

TaskRegistry = seqio.TaskRegistry



def gen_dataset(split, shuffle=False, seed=None, column="text", dataset_params=None):
    dataset = load_dataset(**dataset_params)
    if shuffle:
        if seed:
            dataset = dataset.shuffle(seed=seed)
        else:
            dataset = dataset.shuffle()
    while True:
        for item in dataset[str(split)]:
            yield item[column]


def dataset_fn(split, shuffle_files, seed=None, dataset_params=None):
    return tf.data.Dataset.from_generator(
        functools.partial(gen_dataset, split, shuffle_files, seed, dataset_params=dataset_params),
        output_signature=tf.TensorSpec(shape=(), dtype=tf.string, name=dataset_name)
    )


@utils.map_over_dataset
def target_to_key(x, key_map, target_key):
    """Assign the value from the dataset to target_key in key_map"""
    return {**key_map, target_key: x}



dataset_name = 'oscar-corpus/OSCAR-2109'
subset= 'mr'
dataset_params = {"path": dataset_name, "language":subset, "use_auth_token":True}
dataset_shapes = None

TaskRegistry.add(
    "oscar_marathi_corpus",
    source=seqio.FunctionDataSource(
        dataset_fn=functools.partial(dataset_fn, dataset_params=dataset_params),
        splits=("train", "validation"),
        caching_permitted=False,
        num_input_examples=dataset_shapes,
    ),
preprocessors=[
functools.partial(
target_to_key, key_map={
"targets": None,
}, target_key="targets")],
    output_features={"targets": seqio.Feature(vocabulary=seqio.PassThroughVocabulary, add_eos=False, dtype=tf.string, rank=0)},
    metric_fns=[]
)

dataset = seqio.get_mixture_or_task("oscar_marathi_corpus").get_dataset(
    sequence_length=None,
    split="train",
    shuffle=True,
    num_epochs=1,
    shard_info=seqio.ShardInfo(index=0, num_shards=10),
    use_cached=False,
    seed=42
)
for _, ex in zip(range(5), dataset):
     print(ex['targets'].numpy().decode())
```


### Owner

_No response_
{'url': 'https://api.github.com/repos/huggingface/datasets/issues/4417/reactions', 'total_count': 0, '+1': 0, '-1': 0, 'laugh': 0, 'hooray': 0, 'confused': 0, 'heart': 0, 'rocket': 0, 'eyes': 0}