Remove checksum download in c4
{'login': 'lhoestq', 'id': 42851186, 'node_id': 'MDQ6VXNlcjQyODUxMTg2', 'avatar_url': 'https://avatars.githubusercontent.com/u/42851186?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/lhoestq', 'html_url': 'https://github.com/lhoestq', 'followers_url': 'https://api.github.com/users/lhoestq/followers', 'following_url': 'https://api.github.com/users/lhoestq/following{/other_user}', 'gists_url': 'https://api.github.com/users/lhoestq/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/lhoestq/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/lhoestq/subscriptions', 'organizations_url': 'https://api.github.com/users/lhoestq/orgs', 'repos_url': 'https://api.github.com/users/lhoestq/repos', 'events_url': 'https://api.github.com/users/lhoestq/events{/privacy}', 'received_events_url': 'https://api.github.com/users/lhoestq/received_events', 'type': 'User', 'site_admin': False}
[]
[]
['Commenting again in case [previous thread](https://github.com/huggingface/nlp/pull/233) was inactive.\r\n\r\n@lhoestq I am facing `IsADirectoryError` while downloading with this command.\r\nCan you pls look into it & help me.\r\nI\'m using version 0.4.0 of `nlp`.\r\n\r\n```\r\ndataset = load_dataset("c4", \'en\', data_dir=\'.\', beam_runner=\'DirectRunner\')\r\n```\r\n\r\nHere\'s the complete stack trace.\r\n\r\n```\r\nDownloading and preparing dataset c4/en (download: Unknown size, generated: Unknown size, post-processed: Unknown sizetotal: Unknown size) to /home/devops/.cache/huggingface/datasets/c4/en/2.3.0/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7...\r\n\r\n---------------------------------------------------------------------------\r\nIsADirectoryError                         Traceback (most recent call last)\r\n<ipython-input-11-f622e6705e03> in <module>\r\n----> 1 dataset = load_dataset("c4", \'en\', data_dir=\'.\', beam_runner=\'DirectRunner\')\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/load.py in load_dataset(path, name, version, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, **config_kwargs)\r\n    547     # Download and prepare data\r\n    548     builder_instance.download_and_prepare(\r\n--> 549         download_config=download_config, download_mode=download_mode, ignore_verifications=ignore_verifications,\r\n    550     )\r\n    551 \r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)\r\n    461                 if not downloaded_from_gcs:\r\n    462                     self._download_and_prepare(\r\n--> 463                         dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs\r\n    464                     )\r\n    465                 # Sync info\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos)\r\n    964         pipeline = beam_utils.BeamPipeline(runner=beam_runner, options=beam_options,)\r\n    965         super(BeamBasedBuilder, self)._download_and_prepare(\r\n--> 966             dl_manager, verify_infos=False, pipeline=pipeline,\r\n    967         )  # TODO handle verify_infos in beam datasets\r\n    968         # Run pipeline\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)\r\n    516         split_dict = SplitDict(dataset_name=self.name)\r\n    517         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)\r\n--> 518         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)\r\n    519         # Checksums verification\r\n    520         if verify_infos:\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/datasets/c4/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7/c4.py in _split_generators(self, dl_manager, pipeline)\r\n    187         if self.config.realnewslike:\r\n    188             files_to_download["realnews_domains"] = _REALNEWS_DOMAINS_URL\r\n--> 189         file_paths = dl_manager.download_and_extract(files_to_download)\r\n    190 \r\n    191         if self.config.webtextlike:\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download_and_extract(self, url_or_urls)\r\n    218             extracted_path(s): `str`, extracted paths of given URL(s).\r\n    219         """\r\n--> 220         return self.extract(self.download(url_or_urls))\r\n    221 \r\n    222     def get_recorded_sizes_checksums(self):\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download(self, url_or_urls)\r\n    156             lambda url: cached_path(url, download_config=self._download_config,), url_or_urls,\r\n    157         )\r\n--> 158         self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)\r\n    159         return downloaded_path_or_paths\r\n    160 \r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in _record_sizes_checksums(self, url_or_urls, downloaded_path_or_paths)\r\n    106         flattened_downloaded_path_or_paths = flatten_nested(downloaded_path_or_paths)\r\n    107         for url, path in zip(flattened_urls_or_urls, flattened_downloaded_path_or_paths):\r\n--> 108             self._recorded_sizes_checksums[url] = get_size_checksum_dict(path)\r\n    109 \r\n    110     def download_custom(self, url_or_urls, custom_download):\r\n\r\n/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/info_utils.py in get_size_checksum_dict(path)\r\n     77     """Compute the file size and the sha256 checksum of a file"""\r\n     78     m = sha256()\r\n---> 79     with open(path, "rb") as f:\r\n     80         for chunk in iter(lambda: f.read(1 << 20), b""):\r\n     81             m.update(chunk)\r\n\r\nIsADirectoryError: [Errno 21] Is a directory: \'/\'\r\n\r\n```\r\n\r\nCan anyone please try to see what I am doing wrong or is this a bug?']
{'url': 'https://api.github.com/repos/huggingface/datasets/pulls/250', 'html_url': 'https://github.com/huggingface/datasets/pull/250', 'diff_url': 'https://github.com/huggingface/datasets/pull/250.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/250.patch', 'merged_at': datetime.datetime(2020, 6, 8, 9, 16, 59)}
There was a line from the original tfds script that was still there and causing issues when loading the c4 script. This one should fix #233 and allow anyone to load the c4 script to generate the dataset
{'url': 'https://api.github.com/repos/huggingface/datasets/issues/250/reactions', 'total_count': 0, '+1': 0, '-1': 0, 'laugh': 0, 'hooray': 0, 'confused': 0, 'heart': 0, 'rocket': 0, 'eyes': 0}