<commit_before># Lint as: python3
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""NestedMap dict structure."""

import re
from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,
                    Tuple, TypeVar, Union)
import lingvo.compat as tf

_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*')
_SQUARE_BRACKET_PATTERN = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\[(\d+)\]')


NestedMapT = TypeVar('NestedMapT', bound='NestedMap')


class NestedMap(Dict[str, Any]):
  """A simple helper to maintain a dict.

  It is a sub-class of dict with the following extensions/restrictions:
    - It supports attr access to its members (see examples below).
    - Member keys have to be valid identifiers.

  E.g.::

      >>> foo = NestedMap()
      >>> foo['x'] = 10
      >>> foo.y = 20
      >>> assert foo.x * 2 == foo.y
  """

  # Disable pytype attribute checking.
  _HAS_DYNAMIC_ATTRIBUTES = True
  # keys in this list are not allowed in a NestedMap.
  _RESERVED_KEYS = frozenset(dir(dict))
  # sentinel value for deleting keys used in Filter.
  _DELETE = object()

  def __init__(self, *args, **kwargs) -> None:
    super().__init__(*args, **kwargs)
    for key in self.keys():
      assert isinstance(key, str), (
          'Key in a NestedMap has to be a six.string_types. Currently type: %s,'
          ' value: %s' % (str(type(key)), str(key)))
      NestedMap.CheckKey(key)
      assert key not in NestedMap._RESERVED_KEYS, ('%s is a reserved key' % key)

  def __setitem__(self, key: str, value: Any) -> None:
    # Make sure key is a valid expression and is not one of the reserved
    # attributes.
    assert isinstance(
        key,
        str), ('Key in a NestedMap has to be a string type. Current type: %s, '
               'value: %s' % (str(type(key)), str(key)))
    NestedMap.CheckKey(key)
    assert key not in NestedMap._RESERVED_KEYS, ('%s is a reserved key' % key)
    super().__setitem__(key, value)

  def __setattr__(self, name: str, value: Any) -> None:
    self.__setitem__(name, value)

  def __getattr__(self, name: str) -> Any:
    try:
      return self[name]
    except KeyError as e:
      raise AttributeError('%s; available attributes: %s' %
                           (e, sorted(list(self.keys()))))

  def __delattr__(self, name: str) -> None:
    try:
      del self[name]
    except KeyError as e:
      raise AttributeError('%s; available attributes: %s' %
                           (e, sorted(list(self.keys()))))

  def copy(self: NestedMapT) -> NestedMapT:  # pylint: disable=invalid-name
    # Don't delegate w/ super: dict.copy() -> dict.
    return type(self)(self)

  def __deepcopy__(self: NestedMapT, unused_memo) -> NestedMapT:
    """Deep-copies the structure but not the leaf objects."""
    return self.DeepCopy()

  def DeepCopy(self: NestedMapT) -> NestedMapT:
    """Deep-copies the structure but not the leaf objects."""
    return self.Pack(self.Flatten())

  @staticmethod
  def FromNestedDict(
      x: Union[NestedMapT, Mapping[str, Any], List[Any], Tuple[Any, ...]]
  ) -> NestedMapT:
    """Converts every dict in nested structure 'x' to a NestedMap."""
    if isinstance(x, dict):
      res = NestedMap()
      for k, v in x.items():
        res[k] = NestedMap.FromNestedDict(v)
      return res
    elif isinstance(x, (list, tuple)):
      return type(x)(NestedMap.FromNestedDict(v) for v in x)
    else:
      return x
<commit_msg>Include list of attributes in error message for __getitem__ as well (access through [] operator)<commit_after># Lint as: python3
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""NestedMap dict structure."""

import re
from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,
                    Tuple, TypeVar, Union)
import lingvo.compat as tf

_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*')
_SQUARE_BRACKET_PATTERN = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\[(\d+)\]')


NestedMapT = TypeVar('NestedMapT', bound='NestedMap')


class NestedMap(Dict[str, Any]):
  """A simple helper to maintain a dict.

  It is a sub-class of dict with the following extensions/restrictions:
    - It supports attr access to its members (see examples below).
    - Member keys have to be valid identifiers.

  E.g.::

      >>> foo = NestedMap()
      >>> foo['x'] = 10
      >>> foo.y = 20
      >>> assert foo.x * 2 == foo.y
  """

  # Disable pytype attribute checking.
  _HAS_DYNAMIC_ATTRIBUTES = True
  # keys in this list are not allowed in a NestedMap.
  _RESERVED_KEYS = frozenset(dir(dict))
  # sentinel value for deleting keys used in Filter.
  _DELETE = object()

  def __init__(self, *args, **kwargs) -> None:
    super().__init__(*args, **kwargs)
    for key in self.keys():
      assert isinstance(key, str), (
          'Key in a NestedMap has to be a six.string_types. Currently type: %s,'
          ' value: %s' % (str(type(key)), str(key)))
      NestedMap.CheckKey(key)
      assert key not in NestedMap._RESERVED_KEYS, ('%s is a reserved key' % key)

  def __setitem__(self, key: str, value: Any) -> None:
    # Make sure key is a valid expression and is not one of the reserved
    # attributes.
    assert isinstance(
        key,
        str), ('Key in a NestedMap has to be a string type. Current type: %s, '
               'value: %s' % (str(type(key)), str(key)))
    NestedMap.CheckKey(key)
    assert key not in NestedMap._RESERVED_KEYS, ('%s is a reserved key' % key)
    super().__setitem__(key, value)

  def __setattr__(self, name: str, value: Any) -> None:
    self[name] = value

  def __getattr__(self, name: str) -> Any:
    try:
      return self[name]
    except KeyError as e:
      raise AttributeError(e)

  def __delattr__(self, name: str) -> None:
    try:
      del self[name]
    except KeyError as e:
      raise AttributeError(e)

  def __getitem__(self, name: str) -> Any:
    try:
      return super().__getitem__(name)
    except KeyError as e:
      raise KeyError('%s; available attributes: %s' %
                     (e, sorted(list(self.keys()))))

  def __delitem__(self, name: str) -> None:
    try:
      super().__delitem__(name)
    except KeyError as e:
      raise KeyError('%s; available attributes: %s' %
                     (e, sorted(list(self.keys()))))

  def copy(self: NestedMapT) -> NestedMapT:  # pylint: disable=invalid-name
    # Don't delegate w/ super: dict.copy() -> dict.
    return type(self)(self)

  def __deepcopy__(self: NestedMapT, unused_memo) -> NestedMapT:
    """Deep-copies the structure but not the leaf objects."""
    return self.DeepCopy()

  def DeepCopy(self: NestedMapT) -> NestedMapT:
    """Deep-copies the structure but not the leaf objects."""
    return self.Pack(self.Flatten())

  @staticmethod
  def FromNestedDict(
      x: Union[NestedMapT, Mapping[str, Any], List[Any], Tuple[Any, ...]]
  ) -> NestedMapT:
    """Converts every dict in nested structure 'x' to a NestedMap."""
    if isinstance(x, dict):
      res = NestedMap()
      for k, v in x.items():
        res[k] = NestedMap.FromNestedDict(v)
      return res
    elif isinstance(x, (list, tuple)):
      return type(x)(NestedMap.FromNestedDict(v) for v in x)
    else:
      return x

{'fraction_non_alphanumeric': 0.1404773156899811, 'fraction_numerical': 0.004253308128544423, 'mean_word_length': 3.13330078125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2727954', 'n_tokens_mistral': 2732, 'n_tokens_neox': 2632, 'n_words': 902}