<commit_before>"""Helper code to find and load JSON files."""

import abc
import collections
import json
import os

from typing import Mapping

from bigbench import benchmark_tasks
import tensorflow.compat.v2 as tf


class JsonUtils(abc.ABCMeta):
  """Base helper class to find and read JSON files."""

  @abc.abstractclassmethod
  def get_subdir_json_file_paths(cls, base_dir: str) -> Mapping[str, str]:
    raise NotImplementedError()

  @abc.abstractclassmethod
  def load_json_file(cls, filepath: str) -> collections.OrderedDict:
    raise NotImplementedError()


class GFileUtils(JsonUtils):
  """Helper class to find and read JSON files using the gfile API."""

  BENCHMARK_TASKS_BASE_DIR = benchmark_tasks.__path__[0]  # pytype: disable=unsupported-operands
  BENCHMARK_TASKS_METADATA_PATH = os.path.join(BENCHMARK_TASKS_BASE_DIR,
                                               "task_metadata.json")

  @classmethod
  def get_subdir_json_file_paths(cls, base_dir=BENCHMARK_TASKS_BASE_DIR):
    # Each BIG-bench JSON task is in a separate dir: <task_name>/task.json.
    dirs = os.listdir(base_dir)
    tasks_dirs = [os.path.join(base_dir, d) for d in dirs]
    # Look for a task.json file in each task dir.
    tasks_dirs = [d for d in tasks_dirs if tf.io.gfile.isdir(d)]
    json_paths = [
        tf.io.gfile.glob(os.path.join(d, "task.json")) for d in tasks_dirs
    ]
    # Programmatic tasks don't have task.json files; remove those from paths.
    json_paths = [p[0] for p in json_paths if p]
    # Key the paths by task name extracted from the dir.
    json_paths = {str(path).split("/")[-2]: str(path) for path in json_paths}
    return json_paths

  @classmethod
  def load_json_file(cls, filepath):
    with tf.io.gfile.GFile(filepath, "rb") as file:
      json_data = json.load(file, object_pairs_hook=collections.OrderedDict)
    return json_data


def get_default_json_utils():
  return GFileUtils
<commit_msg>changes after review<commit_after>"""Helper code to find and load JSON files."""

import abc
import collections
import json
import os

from typing import Mapping

from bigbench import benchmark_tasks
import tensorflow.compat.v2 as tf


class JsonUtils(abc.ABC):
  """Base helper class to find and read JSON files."""

  @abc.abstractmethod
  def get_subdir_json_file_paths(self, base_dir: str) -> Mapping[str, str]:
    """Finds files named 'task.json' in subdirs of given base_dir."""
    raise NotImplementedError()

  @abc.abstractmethod
  def load_json_file(self, filepath: str) -> collections.OrderedDict:
    """Loads a json file into an OrderedDict."""
    raise NotImplementedError()


class GFileUtils(JsonUtils):
  """Helper class to find and read JSON files using the gfile API."""

  BENCHMARK_TASKS_BASE_DIR = benchmark_tasks.__path__[0]  # pytype: disable=unsupported-operands
  BENCHMARK_TASKS_METADATA_PATH = os.path.join(BENCHMARK_TASKS_BASE_DIR,
                                               "task_metadata.json")

  def get_subdir_json_file_paths(self, base_dir=BENCHMARK_TASKS_BASE_DIR):
    # Each BIG-bench JSON task is in a separate dir: <task_name>/task.json.
    dirs = os.listdir(base_dir)
    tasks_dirs = [os.path.join(base_dir, d) for d in dirs]
    # Look for a task.json file in each task dir.
    tasks_dirs = [d for d in tasks_dirs if tf.io.gfile.isdir(d)]
    json_paths = [
        tf.io.gfile.glob(os.path.join(d, "task.json")) for d in tasks_dirs
    ]
    # Programmatic tasks don't have task.json files; remove those from paths.
    json_paths = [p[0] for p in json_paths if p]
    # Key the paths by task name extracted from the dir.
    json_paths = {str(path).split("/")[-2]: str(path) for path in json_paths}
    return json_paths

  def load_json_file(self, filepath):
    with tf.io.gfile.GFile(filepath, "rb") as file:
      json_data = json.load(file, object_pairs_hook=collections.OrderedDict)
    return json_data


def get_default_json_utils():
  return GFileUtils()

{'fraction_non_alphanumeric': 0.09650455927051672, 'fraction_numerical': 0.002026342451874367, 'mean_word_length': 3.7011904761904764, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6058020', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1307, 'n_words': 374}