<commit_before>"""Constants relevant to ast code."""

import ast

NODE_TYPE_TO_TOKENS = {
    ast.Add: ('+',),
    ast.Sub: ('-',),
    ast.Mult: ('*',),
    ast.Div: ('/',),
    ast.Mod: ('%',),
    ast.BitAnd: ('&',),
    ast.BitOr: ('|',),
    ast.BitXor: ('^',),
    ast.FloorDiv: ('//',),
    ast.Pow: ('**',),
    ast.LShift: ('<<',),
    ast.RShift: ('>>',),
    ast.BitAnd: ('&',),
    ast.BitOr: ('|',),
    ast.BitXor: ('^',),
    ast.FloorDiv: ('//',),
    ast.Invert: ('~',),
    ast.Not: ('not',),
    ast.UAdd: ('+',),
    ast.USub: ('-',),
    ast.And: ('and',),
    ast.Or: ('or',),
    ast.Eq: ('==',),
    ast.NotEq: ('!=',),
    ast.Lt: ('<',),
    ast.LtE: ('<=',),
    ast.Gt: ('>',),
    ast.GtE: ('>=',),
    ast.Is: ('is',),
    ast.IsNot: ('is', 'not',),
    ast.In: ('in',),
    ast.NotIn: ('not', 'in',),
}


if hasattr(ast, 'MatMult'):
  NODE_TYPE_TO_TOKENS[ast.MatMult] = ('@',)<commit_msg>Sort ast nodes in constants + remove duplicates<commit_after>"""Constants relevant to ast code."""

import ast

NODE_TYPE_TO_TOKENS = {
    ast.Add: ('+',),
    ast.And: ('and',),
    ast.BitAnd: ('&',),
    ast.BitOr: ('|',),
    ast.BitXor: ('^',),
    ast.Div: ('/',),
    ast.Eq: ('==',),
    ast.FloorDiv: ('//',),
    ast.Gt: ('>',),
    ast.GtE: ('>=',),
    ast.In: ('in',),
    ast.Invert: ('~',),
    ast.Is: ('is',),
    ast.IsNot: ('is', 'not',),
    ast.LShift: ('<<',),
    ast.Lt: ('<',),
    ast.LtE: ('<=',),
    ast.Mod: ('%',),
    ast.Mult: ('*',),
    ast.Not: ('not',),
    ast.NotEq: ('!=',),
    ast.NotIn: ('not', 'in',),
    ast.Or: ('or',),
    ast.Pow: ('**',),
    ast.RShift: ('>>',),
    ast.Sub: ('-',),
    ast.UAdd: ('+',),
    ast.USub: ('-',),
}


if hasattr(ast, 'MatMult'):
  NODE_TYPE_TO_TOKENS[ast.MatMult] = ('@',)
{'fraction_non_alphanumeric': 0.3461104847801578, 'fraction_numerical': 0.0, 'mean_word_length': 2.6077235772357725, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7120780', 'n_tokens_mistral': 808, 'n_tokens_neox': 800, 'n_words': 108}