<commit_before>import tokenizer

class Tree:
  def __init__(self, elements):
    self._elements = elements

  def count(self):
    return len(self.elements())

  def elements(self):
    result = []
    for element in self._elements:
      if element.__class__ == Tree:
        result += [element.elements()]
      elif element.isspace() or element.startswith("#"):
        continue
      else:
        result += [element]
    return result

  def to_string(self):
    result = ''
    for element in self._elements:
      if element.__class__ == Tree:
        result += '('
        result += element.to_string()
        result += ')'
      else:
        result += element
    return result

def read_expression(tokenizer1):
    while True:
        token = tokenizer1.next_token()
        if token == None:
            return None
        if token == '(':
            result = []
            while True:
                subsexp = read_expression(tokenizer1)
                if subsexp == ')':
                    break
                elif subsexp == None:
                    raise SyntaxError('eof inside sexp')
                result.append(subsexp)
            return Tree(result)
        else:
            return token

def parse(stream):
  tokenizer1 = tokenizer.Tokenizer(stream)
  tokens = []
  while True:
    expression = read_expression(tokenizer1)
    if expression == None:
      break
    tokens += [expression]
  return Tree(tokens)
<commit_msg>Rename poorly named variable.<commit_after>import tokenizer

class Tree:
  def __init__(self, elements):
    self._elements = elements

  def count(self):
    return len(self.elements())

  def elements(self):
    result = []
    for element in self._elements:
      if element.__class__ == Tree:
        result += [element.elements()]
      elif element.isspace() or element.startswith("#"):
        continue
      else:
        result += [element]
    return result

  def to_string(self):
    result = ''
    for element in self._elements:
      if element.__class__ == Tree:
        result += '('
        result += element.to_string()
        result += ')'
      else:
        result += element
    return result

def read_expression(tokenizer1):
    while True:
        token = tokenizer1.next_token()
        if token == None:
            return None
        if token == '(':
            result = []
            while True:
                subsexp = read_expression(tokenizer1)
                if subsexp == ')':
                    break
                elif subsexp == None:
                    raise SyntaxError('eof inside sexp')
                result.append(subsexp)
            return Tree(result)
        else:
            return token

def parse(stream):
  tokenizer1 = tokenizer.Tokenizer(stream)
  expressions = []
  while True:
    expression = read_expression(tokenizer1)
    if expression == None:
      break
    expressions.append(expression)
  return Tree(expressions)

{'fraction_non_alphanumeric': 0.10033898305084746, 'fraction_numerical': 0.003389830508474576, 'mean_word_length': 1.8988212180746562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2661831', 'n_tokens_mistral': 803, 'n_tokens_neox': 790, 'n_words': 212}