<commit_before># coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r"""Update checksums to contain filename."""

import os
import pathlib
import requests
from absl import app
from typing import Dict, List
from concurrent import futures

import tensorflow_datasets as tfds
from tensorflow_datasets.core.download import checksums


TFDS_PATH = tfds.core.utils.tfds_write_path()


def _collect_path_to_update() -> List[tfds.core.utils.ReadWritePath]:
  """Collect checksums paths to update."""
  url_checksums_paths = list(checksums._checksum_paths().values())

  builder_checksums_path = []
  for name in tfds.list_builders():
    builder_cls = tfds.builder_cls(name)
    if builder_cls.url_infos != None:
      builder_checksums_path.append(builder_cls._checksums_path)
  return url_checksums_paths + builder_checksums_path

def _request_filename(url) -> (str, str):
  """Get filename of dataset at `url`."""
  filename  = ''
  try:
    response = requests.get(url, timeout=10)
    filename = tfds.core.download.downloader._get_filename(response)
    print(f'Success for {url}')
  except requests.exceptions.HTTPError as http_err:
    print(f'HTTP Error {http_err} for {url}.')
  return (url, filename)

def _update_url_infos(url_infos, filenames) -> Dict[str, checksums.UrlInfo]:
  """Get and update dataset filname in UrlInfo."""
  for url, filename in filenames:
    url_infos[url].filename = filename
  return url_infos


def main(_):
  with futures.ThreadPoolExecutor(max_workers=100) as executor:
    future_filenames = {}
    cached_url_infos = {}
    # Collect legacy datasets as well as new datasets
    for url_path in _collect_path_to_update():
      url_infos = checksums.load_url_infos(url_path)
      cached_url_infos[url_path] = url_infos
      future_filenames[url_path] = executor.map(_request_filename, url_infos)
    for path, future_filename in future_filenames.items():
      old_url_infos = cached_url_infos[path]
      updated_url_infos = _update_url_infos(old_url_infos, future_filename)
      checksums.save_url_infos(path, updated_url_infos)


if __name__ == '__main__':
  app.run(main)<commit_msg>Remove duplicate urls<commit_after># coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r"""Update checksums to contain filename."""

import os
import pathlib
import requests
import logging
from absl import app
from typing import Dict, List
from concurrent import futures

import tensorflow_datasets as tfds
from tensorflow_datasets.core import utils
from tensorflow_datasets.core.download import checksums

TFDS_PATH = tfds.core.utils.tfds_write_path()


def _collect_path_to_url_infos() -> Dict[utils.ReadWritePath,
                                          Dict[str, checksums.UrlInfo]]:
  """Collect checksums paths to url_infos."""
  path_to_url_infos = {}

  url_checksums_paths = list(checksums._checksum_paths().values())

  builder_checksums_path = []
  for name in tfds.list_builders():
    builder_cls = tfds.builder_cls(name)
    if builder_cls.url_infos is not None:
      builder_checksums_path.append(builder_cls._checksums_path)

  for url_path in url_checksums_paths + builder_checksums_path:
    path_to_url_infos[url_path] = checksums.load_url_infos(url_path)
  return path_to_url_infos

def _request_filename(url) -> str:
  """Get filename of dataset at `url`."""
  filename  = ''
  try:
    response = requests.get(url, timeout=10)
    filename = tfds.core.download.downloader._get_filename(response)
    print(f'Success for {url}')
  except requests.exceptions.HTTPError as http_err:
    print(f'HTTP Error {http_err} for {url}.')
  return filename

def _update_url_infos(url_infos, urls_to_filename) -> Dict[str, checksums.UrlInfo]:
  """Get and update dataset filname in UrlInfo."""
  for url, url_info in url_infos.items():
    new_filename = urls_to_filename[url].result()
    old_filename = url_info.filename
    if old_filename and old_filename != new_filename:
      logging.warning(f"Filename for {url} already exist. "
                      f"Updating {old_filename} to {new_filename}")
    url_info.filename = new_filename
  return url_infos

def main(_):
  path_to_url_infos = _collect_path_to_url_infos()

  # Remove duplicate urls
  all_url_infos = {}
  for url_infos in path_to_url_infos.values():
    all_url_infos.update(url_infos)

  with futures.ThreadPoolExecutor(max_workers=100) as executor:
    urls_to_filename = {
      url: executor.submit(_request_filename, url)
      for url in all_url_infos
    }
    for path, url_infos in path_to_url_infos.items():
      url_infos = _update_url_infos(url_infos, urls_to_filename)
      checksums.save_url_infos(path, url_infos)


if __name__ == '__main__':
  app.run(main)
{'fraction_non_alphanumeric': 0.08739872435786933, 'fraction_numerical': 0.004826754007929667, 'mean_word_length': 4.447887323943662, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1691788', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1822, 'n_words': 510}