<commit_before>"""GCL query module.

This is a bit haphazard for now, but it defines GPath (yes, very creative),
which supports selection of elements in an object tree.
"""


import pyparsing as p

import gcl


class QueryError(RuntimeError):
  pass

def bla(x):
  return tuple(x)

variable = gcl.variable.copy().setParseAction(lambda x: str(x[0]))
alts = gcl.bracketedList('{', '}', ',', variable, bla)
list_index = gcl.sym('[') + gcl.integer.copy().setParseAction(lambda x: int(x[0])) + gcl.sym(']')
everything = p.Literal('*')
element = variable | alts | list_index | everything

selector = p.Group(p.Optional(element + p.ZeroOrMore(gcl.sym('.') + element)))


def parseSelector(s):
  return selector.parseString(s, parseAll=True)[0]


def listKey(ix):
  return '[%d]' % ix


def isListKey(key):
  return key.startswith('[')



def is_tuple(x):
  return isinstance(x, gcl.Tuple) or isinstance(x, dict)


class GPath(object):
  """Path selector specifications

  Format:

    name1.name2.*.name3.{name5,name6,name7}<commit_msg>More professional function name<commit_after>"""GCL query module.

This is a bit haphazard for now, but it defines GPath (yes, very creative),
which supports selection of elements in an object tree.
"""


import pyparsing as p

import gcl


class QueryError(RuntimeError):
  pass

def mkAlternates(x):
  return tuple(x)

variable = gcl.variable.copy().setParseAction(lambda x: str(x[0]))
alts = gcl.bracketedList('{', '}', ',', variable, mkAlternates)
list_index = gcl.sym('[') + gcl.integer.copy().setParseAction(lambda x: int(x[0])) + gcl.sym(']')
everything = p.Literal('*')
element = variable | alts | list_index | everything

selector = p.Group(p.Optional(element + p.ZeroOrMore(gcl.sym('.') + element)))


def parseSelector(s):
  return selector.parseString(s, parseAll=True)[0]


def listKey(ix):
  return '[%d]' % ix


def isListKey(key):
  return key.startswith('[')



def is_tuple(x):
  return isinstance(x, gcl.Tuple) or isinstance(x, dict)


class GPath(object):
  """Path selector specifications

  Format:

    name1.name2.*.name3.{name5,name6,name7}
{'fraction_non_alphanumeric': 0.15663801337153774, 'fraction_numerical': 0.008595988538681949, 'mean_word_length': 4.918079096045198, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4063374', 'n_tokens_mistral': 758, 'n_tokens_neox': 726, 'n_words': 188}