Make `ted_talks_iwslt` dataset streamable
{'login': 'mariosasko', 'id': 47462742, 'node_id': 'MDQ6VXNlcjQ3NDYyNzQy', 'avatar_url': 'https://avatars.githubusercontent.com/u/47462742?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/mariosasko', 'html_url': 'https://github.com/mariosasko', 'followers_url': 'https://api.github.com/users/mariosasko/followers', 'following_url': 'https://api.github.com/users/mariosasko/following{/other_user}', 'gists_url': 'https://api.github.com/users/mariosasko/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/mariosasko/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/mariosasko/subscriptions', 'organizations_url': 'https://api.github.com/users/mariosasko/orgs', 'repos_url': 'https://api.github.com/users/mariosasko/repos', 'events_url': 'https://api.github.com/users/mariosasko/events{/privacy}', 'received_events_url': 'https://api.github.com/users/mariosasko/received_events', 'type': 'User', 'site_admin': False}
[]
[]
['Thanks for adding this @mariosasko! It worked for me when running it with a local data file, however, when using the file on Google Drive I get the following error:\r\n```Python\r\nds = load_dataset("./ted_talks_iwslt","eu_ca_2014", streaming=True, split="train", use_auth_token=True)\r\nnext(iter(ds))\r\n```\r\n```\r\n---------------------------------------------------------------------------\r\nClientResponseError                       Traceback (most recent call last)\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:383, in HTTPFileSystem._info(self, url, **kwargs)\r\n    381 try:\r\n    382     info.update(\r\n--> 383         await _file_info(\r\n    384             url,\r\n    385             size_policy=policy,\r\n    386             session=session,\r\n    387             **self.kwargs,\r\n    388             **kwargs,\r\n    389         )\r\n    390     )\r\n    391     if info.get("size") is not None:\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:734, in _file_info(url, session, size_policy, **kwargs)\r\n    733 async with r:\r\n--> 734     r.raise_for_status()\r\n    736     # TODO:\r\n    737     #  recognise lack of \'Accept-Ranges\',\r\n    738     #                 or \'Accept-Ranges\': \'none\' (not \'bytes\')\r\n    739     #  to mean streaming only, no random access => return None\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/aiohttp/client_reqrep.py:1004, in ClientResponse.raise_for_status(self)\r\n   1003 self.release()\r\n-> 1004 raise ClientResponseError(\r\n   1005     self.request_info,\r\n   1006     self.history,\r\n   1007     status=self.status,\r\n   1008     message=self.reason,\r\n   1009     headers=self.headers,\r\n   1010 )\r\n\r\nClientResponseError: 403, message=\'Forbidden\', url=URL(\'https://drive.google.com/u/0/uc?id=1Cz1Un9p8Xn9IpEMMrg2kXSDt0dnjxc4z&export=download&confirm=1RJz\')\r\n\r\nThe above exception was the direct cause of the following exception:\r\n\r\nFileNotFoundError                         Traceback (most recent call last)\r\nInput In [9], in <module>\r\n      1 iterable = iter(ds)\r\n      2 for i in range(10):\r\n----> 3     item = next(iterable)\r\n      4     print(item[\'text\'][:10], item[\'meta\'])\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/iterable_dataset.py:341, in IterableDataset.__iter__(self)\r\n    340 def __iter__(self):\r\n--> 341     for key, example in self._iter():\r\n    342         if self.features:\r\n    343             # we encode the example for ClassLabel feature types for example\r\n    344             encoded_example = self.features.encode_example(example)\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/iterable_dataset.py:338, in IterableDataset._iter(self)\r\n    336 else:\r\n    337     ex_iterable = self._ex_iterable\r\n--> 338 yield from ex_iterable\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/iterable_dataset.py:78, in ExamplesIterable.__iter__(self)\r\n     77 def __iter__(self):\r\n---> 78     for key, example in self.generate_examples_fn(**self.kwargs):\r\n     79         yield key, example\r\n\r\nFile ~/.cache/huggingface/modules/datasets_modules/datasets/lm_en_ted_talks_iwslt/756148758e86e64a350f9b320744a2bd5ed5cff74f7df620763a2b5e1a45e6c6/lm_en_ted_talks_iwslt.py:118, in TedTalksIWSLT._generate_examples(self, files)\r\n    116 for _LANG in _LANG_CODES:\r\n    117     source_file_path = _YEAR_FOLDER[year] + "/ted_" + _LANG + _YEAR[year] + ".zip"\r\n--> 118     for path, file in files:\r\n    119         if path.endswith(source_file_path):\r\n    120             source_talks, _ = parse_zip_file(path, file.read())\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/utils/streaming_download_manager.py:596, in StreamingDownloadManager.iter_archive(self, urlpath_or_buf)\r\n    594     yield from _iter_archive(urlpath_or_buf)\r\n    595 else:\r\n--> 596     with xopen(urlpath_or_buf, "rb", use_auth_token=self.download_config.use_auth_token) as f:\r\n    597         yield from _iter_archive(f)\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/datasets/utils/streaming_download_manager.py:296, in xopen(file, mode, use_auth_token, *args, **kwargs)\r\n    294     new_kwargs = {}\r\n    295 kwargs = {**kwargs, **new_kwargs}\r\n--> 296 file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()\r\n    297 _add_retries_to_file_obj_read_method(file_obj)\r\n    298 return file_obj\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/core.py:140, in OpenFile.open(self)\r\n    132 def open(self):\r\n    133     """Materialise this as a real open file without context\r\n    134 \r\n    135     The file should be explicitly closed to avoid enclosed file\r\n   (...)\r\n    138     been deleted; but a with-context is better style.\r\n    139     """\r\n--> 140     out = self.__enter__()\r\n    141     closer = out.close\r\n    142     fobjects = self.fobjects.copy()[:-1]\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/core.py:103, in OpenFile.__enter__(self)\r\n    100 def __enter__(self):\r\n    101     mode = self.mode.replace("t", "").replace("b", "") + "b"\r\n--> 103     f = self.fs.open(self.path, mode=mode)\r\n    105     self.fobjects = [f]\r\n    107     if self.compression is not None:\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/spec.py:1009, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)\r\n   1007 else:\r\n   1008     ac = kwargs.pop("autocommit", not self._intrans)\r\n-> 1009     f = self._open(\r\n   1010         path,\r\n   1011         mode=mode,\r\n   1012         block_size=block_size,\r\n   1013         autocommit=ac,\r\n   1014         cache_options=cache_options,\r\n   1015         **kwargs,\r\n   1016     )\r\n   1017     if compression is not None:\r\n   1018         from fsspec.compression import compr\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:343, in HTTPFileSystem._open(self, path, mode, block_size, autocommit, cache_type, cache_options, size, **kwargs)\r\n    341 kw["asynchronous"] = self.asynchronous\r\n    342 kw.update(kwargs)\r\n--> 343 size = size or self.info(path, **kwargs)["size"]\r\n    344 session = sync(self.loop, self.set_session)\r\n    345 if block_size and size:\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/asyn.py:91, in sync_wrapper.<locals>.wrapper(*args, **kwargs)\r\n     88 @functools.wraps(func)\r\n     89 def wrapper(*args, **kwargs):\r\n     90     self = obj or args[0]\r\n---> 91     return sync(self.loop, func, *args, **kwargs)\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/asyn.py:71, in sync(loop, func, timeout, *args, **kwargs)\r\n     69     raise FSTimeoutError from return_result\r\n     70 elif isinstance(return_result, BaseException):\r\n---> 71     raise return_result\r\n     72 else:\r\n     73     return return_result\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/asyn.py:25, in _runner(event, coro, result, timeout)\r\n     23     coro = asyncio.wait_for(coro, timeout=timeout)\r\n     24 try:\r\n---> 25     result[0] = await coro\r\n     26 except Exception as ex:\r\n     27     result[0] = ex\r\n\r\nFile ~/git/bigscience-datasets/env/lib/python3.9/site-packages/fsspec/implementations/http.py:396, in HTTPFileSystem._info(self, url, **kwargs)\r\n    393     except Exception as exc:\r\n    394         if policy == "get":\r\n    395             # If get failed, then raise a FileNotFoundError\r\n--> 396             raise FileNotFoundError(url) from exc\r\n    397         logger.debug(str(exc))\r\n    399 return {"name": url, "size": None, **info, "type": "file"}\r\n\r\nFileNotFoundError: https://drive.google.com/u/0/uc?id=1Cz1Un9p8Xn9IpEMMrg2kXSDt0dnjxc4z&export=download&confirm=1RJz\r\n```']
{'url': 'https://api.github.com/repos/huggingface/datasets/pulls/3635', 'html_url': 'https://github.com/huggingface/datasets/pull/3635', 'diff_url': 'https://github.com/huggingface/datasets/pull/3635.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/3635.patch', 'merged_at': None}
{'url': 'https://api.github.com/repos/huggingface/datasets/issues/3635/reactions', 'total_count': 0, '+1': 0, '-1': 0, 'laugh': 0, 'hooray': 0, 'confused': 0, 'heart': 0, 'rocket': 0, 'eyes': 0}