<commit_before># Copyright 2021 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Data classes for job-related abstractions."""

import abc
import itertools
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, TypeVar, Union

import attr

from xmanager.xm import pattern_matching

UserArgs = Union[Mapping, Sequence]


class SequentialArgs:
  """A sequence of arguments supporting keyword-based value updates."""

  @attr.s(auto_attribs=True)
  class _RegularItem:
    value: Any

  @attr.s(auto_attribs=True)
  class _KeywordItem:
    name: str

  def __init__(self):
    self._items: List[Union[SequentialArgs._RegularItem,
                            SequentialArgs._KeywordItem]] = []
    self._kwvalues: Dict[str, Any] = {}

  def _ingest_regular_item(self, value: Any) -> None:
    self._items.append(SequentialArgs._RegularItem(value))

  def _ingest_keyword_item(self, name: str, value: Any) -> None:
    if name not in self._kwvalues:
      self._items.append(SequentialArgs._KeywordItem(name))
    self._kwvalues[name] = value

  @staticmethod
  def merge(operands: Iterable['SequentialArgs']) -> 'SequentialArgs':
    """Merges several instances into one left-to-right."""
    result = SequentialArgs()

    def import_regular_item(item: SequentialArgs._RegularItem,
                            _: SequentialArgs):
      result._ingest_regular_item(item.value)  # pylint: disable=protected-access

    def import_keyword_item(item: SequentialArgs._KeywordItem,
                            operand: SequentialArgs):
      result._ingest_keyword_item(item.name, operand._kwvalues[item.name])  # pylint: disable=protected-access

    for operand in operands:
      matcher = pattern_matching.match(
          import_regular_item,
          import_keyword_item,
      )
      for item in operand._items:  # pylint: disable=protected-access
        matcher(item, operand)
    return result

  @staticmethod
  def from_collection(collection: UserArgs) -> 'SequentialArgs':
    """Populates a new instance from a given collection."""
    result = SequentialArgs()

    def import_mapping(collection: Mapping[Any, Any]) -> None:
      for key, value in collection.items():
        result._ingest_keyword_item(str(key), value)  # pylint: disable=protected-access

    def import_sequence(collection: Sequence[Any]) -> None:
      for value in collection:
        result._ingest_regular_item(value)  # pylint: disable=protected-access

    matcher = pattern_matching.match(import_mapping, import_sequence)
    matcher(collection)
    return result

  def to_list(self, escaper: Callable[[Any], str]) -> List[str]:
    """Exports items as a list ready to be passed into the command line."""

    def export_regular_item(item: SequentialArgs._RegularItem) -> List[str]:
      return [escaper(item.value)]

    def export_keyword_item(item: SequentialArgs._KeywordItem) -> List[str]:
      return [escaper(f'--{item.name}'), escaper(self._kwvalues[item.name])]

    matcher = pattern_matching.match(
        export_regular_item,
        export_keyword_item,
    )
    return list(itertools.chain(*[matcher(item) for item in self._items]))

  def __eq__(self, other) -> bool:
    return isinstance(other, SequentialArgs) and all([
        self._items == other._items,
        self._kwvalues == other._kwvalues,
    ])

  def __repr__(self) -> str:
    return f"[{', '.join(self.to_list(repr))}]"
<commit_msg>Fix Gcp run.<commit_after># Copyright 2021 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Data classes for job-related abstractions."""

import abc
import itertools
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, TypeVar, Union

import attr

from xmanager.xm import pattern_matching

UserArgs = Union[Mapping, Sequence]


class SequentialArgs:
  """A sequence of arguments supporting keyword-based value updates."""

  @attr.s(auto_attribs=True)
  class _RegularItem:
    value: Any

  @attr.s(auto_attribs=True)
  class _KeywordItem:
    name: str

  def __init__(self):
    self._items: List[Union[SequentialArgs._RegularItem,
                            SequentialArgs._KeywordItem]] = []
    self._kwvalues: Dict[str, Any] = {}

  def _ingest_regular_item(self, value: Any) -> None:
    self._items.append(SequentialArgs._RegularItem(value))

  def _ingest_keyword_item(self, name: str, value: Any) -> None:
    if name not in self._kwvalues:
      self._items.append(SequentialArgs._KeywordItem(name))
    self._kwvalues[name] = value

  @staticmethod
  def merge(operands: Iterable['SequentialArgs']) -> 'SequentialArgs':
    """Merges several instances into one left-to-right."""
    result = SequentialArgs()

    def import_regular_item(item: SequentialArgs._RegularItem,
                            _: SequentialArgs):
      result._ingest_regular_item(item.value)  # pylint: disable=protected-access

    def import_keyword_item(item: SequentialArgs._KeywordItem,
                            operand: SequentialArgs):
      result._ingest_keyword_item(item.name, operand._kwvalues[item.name])  # pylint: disable=protected-access

    for operand in operands:
      matcher = pattern_matching.match(
          import_regular_item,
          import_keyword_item,
      )
      for item in operand._items:  # pylint: disable=protected-access
        matcher(item, operand)
    return result

  @staticmethod
  def from_collection(collection: UserArgs) -> 'SequentialArgs':
    """Populates a new instance from a given collection."""
    result = SequentialArgs()

    def import_mapping(collection: Mapping[Any, Any]) -> None:
      for key, value in collection.items():
        result._ingest_keyword_item(str(key), value)  # pylint: disable=protected-access

    def import_sequence(collection: Sequence[Any]) -> None:
      for value in collection:
        result._ingest_regular_item(value)  # pylint: disable=protected-access

    matcher = pattern_matching.match(import_mapping, import_sequence)
    matcher(collection)
    return result

  def to_list(self, escaper: Callable[[Any], str]) -> List[str]:
    """Exports items as a list ready to be passed into the command line."""

    def export_regular_item(item: SequentialArgs._RegularItem) -> List[str]:
      return [escaper(item.value)]

    def export_keyword_item(item: SequentialArgs._KeywordItem) -> List[str]:
      return [escaper(f'--{item.name}'), escaper(self._kwvalues[item.name])]

    matcher = pattern_matching.match(
        export_regular_item,
        export_keyword_item,
    )
    return list(itertools.chain(*[matcher(item) for item in self._items]))

  def to_dict(self) -> Dict[str, Any]:
    return self._kwvalues

  def __eq__(self, other) -> bool:
    return isinstance(other, SequentialArgs) and all([
        self._items == other._items,
        self._kwvalues == other._kwvalues,
    ])

  def __repr__(self) -> str:
    return f"[{', '.join(self.to_list(repr))}]"

{'fraction_non_alphanumeric': 0.1032290238124922, 'fraction_numerical': 0.0019947637451689314, 'mean_word_length': 3.674825174825175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '783074', 'n_tokens_mistral': 2377, 'n_tokens_neox': 2315, 'n_words': 646}