Remove unnecessary nested dict
{'login': 'mariamabarham', 'id': 38249783, 'node_id': 'MDQ6VXNlcjM4MjQ5Nzgz', 'avatar_url': 'https://avatars.githubusercontent.com/u/38249783?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/mariamabarham', 'html_url': 'https://github.com/mariamabarham', 'followers_url': 'https://api.github.com/users/mariamabarham/followers', 'following_url': 'https://api.github.com/users/mariamabarham/following{/other_user}', 'gists_url': 'https://api.github.com/users/mariamabarham/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/mariamabarham/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/mariamabarham/subscriptions', 'organizations_url': 'https://api.github.com/users/mariamabarham/orgs', 'repos_url': 'https://api.github.com/users/mariamabarham/repos', 'events_url': 'https://api.github.com/users/mariamabarham/events{/privacy}', 'received_events_url': 'https://api.github.com/users/mariamabarham/received_events', 'type': 'User', 'site_admin': False}
[]
[]
['We can probably scan the dataset scripts with a regexpr to try to identify this pattern cc @patrickvonplaten maybe'
 '@mariamabarham This script should work. I tested it for a couple of datasets. There might be exceptions where the script breaks - did not test everything.\r\n\r\n```python\r\n#!/usr/bin/env python3\r\n\r\nfrom nlp import prepare_module, DownloadConfig, import_main_class, hf_api\r\nimport tempfile\r\n\r\n\r\ndef scan_for_nested_unnecessary_dict(dataset_name):\r\n\r\n    def load_builder_class(dataset_name):\r\n        module_path = prepare_module(dataset_name, download_config=DownloadConfig(force_download=True))\r\n        return import_main_class(module_path)\r\n\r\n    def load_configs(dataset_name):\r\n        builder_cls = load_builder_class(dataset_name)\r\n        if len(builder_cls.BUILDER_CONFIGS) == 0:\r\n            return [None]\r\n        return builder_cls.BUILDER_CONFIGS\r\n\r\n    def scan_features_for_nested_dict(features):\r\n        is_sequence = False\r\n        if hasattr(features, "_type"):\r\n            if features._type != \'Sequence\':\r\n                return False\r\n            else:\r\n                is_sequence = True\r\n                features = features.feature\r\n\r\n        if isinstance(features, list):\r\n            for value in features:\r\n                if scan_features_for_nested_dict(value):\r\n                    return True\r\n            return False\r\n\r\n        elif isinstance(features, dict):\r\n            for key, value in features.items():\r\n                if is_sequence and len(features.keys()) == 1 and hasattr(features[key], "_type") and features[key]._type != "Sequence":\r\n                    return True\r\n                if scan_features_for_nested_dict(value):\r\n                    return True\r\n            return False\r\n        elif hasattr(features, "_type"):\r\n            return False\r\n        else:\r\n            raise ValueError(f"{features} should be either a list, a dict or a feature")\r\n\r\n    configs = load_configs(dataset_name)\r\n\r\n    for config in configs:\r\n        with tempfile.TemporaryDirectory() as processed_temp_dir:\r\n            # create config and dataset\r\n            dataset_builder_cls = load_builder_class(dataset_name)\r\n            name = config.name if config is not None else None\r\n            dataset_builder = dataset_builder_cls(name=name, cache_dir=processed_temp_dir)\r\n\r\n        is_nested_dict_in_dataset = scan_features_for_nested_dict(dataset_builder._info().features)\r\n        if is_nested_dict_in_dataset:\r\n            print(f"{dataset_name} with {name} needs refactoring")\r\n\r\n\r\nif __name__ == "__main__":\r\n    scan_for_nested_unnecessary_dict("race")  # prints True\r\n    scan_for_nested_unnecessary_dict("mlqa")  # prints True\r\n    scan_for_nested_unnecessary_dict("squad")  # prints Nothing\r\n\r\n    # ran the following lines for 1min and seems to work -> didn\'t check for all datasets though\r\n#    api = hf_api.HfApi()\r\n#    all_datasets = [x.id for x in api.dataset_list(with_community_datasets=False)]\r\n#    for dataset in all_datasets:\r\n#        scan_for_nested_unnecessary_dict(dataset)\r\n```'
 '> @mariamabarham This script should work. I tested it for a couple of datasets. There might be exceptions where the script breaks - did not test everything.\r\n> \r\n> ```python\r\n> #!/usr/bin/env python3\r\n> \r\n> from nlp import prepare_module, DownloadConfig, import_main_class, hf_api\r\n> import tempfile\r\n> \r\n> \r\n> def scan_for_nested_unnecessary_dict(dataset_name):\r\n> \r\n>     def load_builder_class(dataset_name):\r\n>         module_path = prepare_module(dataset_name, download_config=DownloadConfig(force_download=True))\r\n>         return import_main_class(module_path)\r\n> \r\n>     def load_configs(dataset_name):\r\n>         builder_cls = load_builder_class(dataset_name)\r\n>         if len(builder_cls.BUILDER_CONFIGS) == 0:\r\n>             return [None]\r\n>         return builder_cls.BUILDER_CONFIGS\r\n> \r\n>     def scan_features_for_nested_dict(features):\r\n>         is_sequence = False\r\n>         if hasattr(features, "_type"):\r\n>             if features._type != \'Sequence\':\r\n>                 return False\r\n>             else:\r\n>                 is_sequence = True\r\n>                 features = features.feature\r\n> \r\n>         if isinstance(features, list):\r\n>             for value in features:\r\n>                 if scan_features_for_nested_dict(value):\r\n>                     return True\r\n>             return False\r\n> \r\n>         elif isinstance(features, dict):\r\n>             for key, value in features.items():\r\n>                 if is_sequence and len(features.keys()) == 1 and hasattr(features[key], "_type") and features[key]._type != "Sequence":\r\n>                     return True\r\n>                 if scan_features_for_nested_dict(value):\r\n>                     return True\r\n>             return False\r\n>         else:\r\n>             raise ValueError(f"{features} should be either a list of a dict")\r\n> \r\n>     configs = load_configs(dataset_name)\r\n> \r\n>     for config in configs:\r\n>         with tempfile.TemporaryDirectory() as processed_temp_dir:\r\n>             # create config and dataset\r\n>             dataset_builder_cls = load_builder_class(dataset_name)\r\n>             name = config.name if config is not None else None\r\n>             dataset_builder = dataset_builder_cls(name=name, cache_dir=processed_temp_dir)\r\n> \r\n>         is_nested_dict_in_dataset = scan_features_for_nested_dict(dataset_builder._info().features)\r\n>         if is_nested_dict_in_dataset:\r\n>             print(f"{dataset_name} with {name} needs refactoring")\r\n> \r\n> \r\n> if __name__ == "__main__":\r\n>     scan_for_nested_unnecessary_dict("race")  # prints True\r\n>     scan_for_nested_unnecessary_dict("mlqa")  # prints True\r\n>     scan_for_nested_unnecessary_dict("squad")  # prints Nothing\r\n> \r\n>     # ran the following lines for 1min and seems to work -> didn\'t check for all datasets though\r\n> #    api = hf_api.HfApi()\r\n> #    all_datasets = [x.id for x in api.dataset_list(with_community_datasets=False)]\r\n> #    for dataset in all_datasets:\r\n> #        scan_for_nested_unnecessary_dict(dataset)\r\n> ```\r\n\r\nGreat, I will try it'
 "I'm not sure the work on this PR was finished @lhoestq  cc @mariamabarham @patrickvonplaten "
 'Sorry for that, apparently there are other datasets that could have unnecessary nested dicts.\r\nWe can have another PR to scan and fix the other datasets.\r\n']
{'url': 'https://api.github.com/repos/huggingface/datasets/pulls/385', 'html_url': 'https://github.com/huggingface/datasets/pull/385', 'diff_url': 'https://github.com/huggingface/datasets/pull/385.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/385.patch', 'merged_at': datetime.datetime(2020, 7, 15, 10, 3, 53)}
This PR is removing unnecessary nested dictionary used in some datasets. For now the following datasets are updated:

- MLQA

- RACE

Will be adding more if necessary.

#378 
{'url': 'https://api.github.com/repos/huggingface/datasets/issues/385/reactions', 'total_count': 0, '+1': 0, '-1': 0, 'laugh': 0, 'hooray': 0, 'confused': 0, 'heart': 0, 'rocket': 0, 'eyes': 0}