<commit_before># Copyright 2015-2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Subtype assigner for lib2to3 trees.

This module assigns extra type information to the lib2to3 trees. This
information is more specific than whether something is an operator or an
identifier. For instance, it can specify if a node in the tree is part of a
subscript.

  AssignSubtypes(): the main function exported by this module.

Annotations:
  subtype: The subtype of a pytree token. See 'format_token' module for a list
      of subtypes.
"""

from lib2to3 import pytree
from lib2to3.pgen2 import token
from lib2to3.pygram import python_symbols as syms

from yapf.yapflib import format_token
from yapf.yapflib import pytree_utils
from yapf.yapflib import pytree_visitor
from yapf.yapflib import style


def AssignSubtypes(tree):
  """Run the subtype assigner visitor over the tree, modifying it in place.

  Arguments:
    tree: the top-level pytree node to annotate with subtypes.
  """
  subtype_assigner = _SubtypeAssigner()
  subtype_assigner.Visit(tree)


# Map tokens in argument lists to their respective subtype.
_ARGLIST_TOKEN_TO_SUBTYPE = {
    '=': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN,
    ':': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN,
    '*': format_token.Subtype.VARARGS_STAR,
    '**': format_token.Subtype.KWARGS_STAR_STAR,
}


class _SubtypeAssigner(pytree_visitor.PyTreeVisitor):
  """_SubtypeAssigner - see file-level docstring for detailed description.

  The subtype is added as an annotation to the pytree token.
  """

  def Visit_dictsetmaker(self, node):  # pylint: disable=invalid-name
    # dictsetmaker ::= (test ':' test (comp_for |
    #                                   (',' test ':' test)* [','])) |
    #                  (test (comp_for | (',' test)* [',']))
    for child in node.children:
      self.Visit(child)

    comp_for = False
    dict_maker = False

    for child in node.children:
      if pytree_utils.NodeName(child) == 'comp_for':
        comp_for = True
        _AppendFirstLeafTokenSubtype(child,
                                     format_token.Subtype.DICT_SET_GENERATOR)
      elif pytree_utils.NodeName(child) in ('COLON', 'DOUBLESTAR'):
        dict_maker = True

    if not comp_for and dict_maker:
      last_was_colon = False
      for child in node.children:
        if dict_maker:
          if pytree_utils.NodeName(child) == "DOUBLESTAR":
            _AppendFirstLeafTokenSubtype(child, format_token.Subtype.KWARGS_STAR_STAR)
          if last_was_colon:
            if style.Get('INDENT_DICTIONARY_VALUE'):
              _InsertPseudoParentheses(child)
            else:
              _AppendFirstLeafTokenSubtype(
                  child, format_token.Subtype.DICTIONARY_VALUE)
          elif (child is not None and (isinstance(child, pytree.Node) or
                                       (not child.value.startswith('#') and
                                        child.value not in '{:,'))):
            # Mark the first leaf of a key entry as a DICTIONARY_KEY. We
            # normally want to split before them if the dictionary cannot exist
            # on a single line.
            _AppendFirstLeafTokenSubtype(child,<commit_msg>Align with project code style<commit_after># Copyright 2015-2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Subtype assigner for lib2to3 trees.

This module assigns extra type information to the lib2to3 trees. This
information is more specific than whether something is an operator or an
identifier. For instance, it can specify if a node in the tree is part of a
subscript.

  AssignSubtypes(): the main function exported by this module.

Annotations:
  subtype: The subtype of a pytree token. See 'format_token' module for a list
      of subtypes.
"""

from lib2to3 import pytree
from lib2to3.pgen2 import token
from lib2to3.pygram import python_symbols as syms

from yapf.yapflib import format_token
from yapf.yapflib import pytree_utils
from yapf.yapflib import pytree_visitor
from yapf.yapflib import style


def AssignSubtypes(tree):
  """Run the subtype assigner visitor over the tree, modifying it in place.

  Arguments:
    tree: the top-level pytree node to annotate with subtypes.
  """
  subtype_assigner = _SubtypeAssigner()
  subtype_assigner.Visit(tree)


# Map tokens in argument lists to their respective subtype.
_ARGLIST_TOKEN_TO_SUBTYPE = {
    '=': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN,
    ':': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN,
    '*': format_token.Subtype.VARARGS_STAR,
    '**': format_token.Subtype.KWARGS_STAR_STAR,
}


class _SubtypeAssigner(pytree_visitor.PyTreeVisitor):
  """_SubtypeAssigner - see file-level docstring for detailed description.

  The subtype is added as an annotation to the pytree token.
  """

  def Visit_dictsetmaker(self, node):  # pylint: disable=invalid-name
    # dictsetmaker ::= (test ':' test (comp_for |
    #                                   (',' test ':' test)* [','])) |
    #                  (test (comp_for | (',' test)* [',']))
    for child in node.children:
      self.Visit(child)

    comp_for = False
    dict_maker = False

    for child in node.children:
      if pytree_utils.NodeName(child) == 'comp_for':
        comp_for = True
        _AppendFirstLeafTokenSubtype(child,
                                     format_token.Subtype.DICT_SET_GENERATOR)
      elif pytree_utils.NodeName(child) in ('COLON', 'DOUBLESTAR'):
        dict_maker = True

    if not comp_for and dict_maker:
      last_was_colon = False
      for child in node.children:
        if dict_maker:
          if pytree_utils.NodeName(child) == 'DOUBLESTAR':
            _AppendFirstLeafTokenSubtype(child, format_token.Subtype.KWARGS_STAR_STAR)
          if last_was_colon:
            if style.Get('INDENT_DICTIONARY_VALUE'):
              _InsertPseudoParentheses(child)
            else:
              _AppendFirstLeafTokenSubtype(
                  child, format_token.Subtype.DICTIONARY_VALUE)
          elif (child is not None and (isinstance(child, pytree.Node) or
                                       (not child.value.startswith('#') and
                                        child.value not in '{:,'))):
            # Mark the first leaf of a key entry as a DICTIONARY_KEY. We
            # normally want to split before them if the dictionary cannot exist
            # on a single line.
            _AppendFirstLeafTokenSubtype(child,
{'fraction_non_alphanumeric': 0.07757931218341775, 'fraction_numerical': 0.006131698213809651, 'mean_word_length': 2.881531298499741, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7638134', 'n_tokens_mistral': 2241, 'n_tokens_neox': 2148, 'n_words': 719}