<commit_before>from ply import *
import bfallex
import sys

class Node:
  def __init__(self, type, *values):
    self.type = type
    self.values = values
    self.count = len(values)

  def __repr__(self):
    return '%s%r'%(self.type, self.values)

  def extend(self, *values):
    self.values += values
    self.count += len(values)

tokens = bfallex.tokens

constants = {
  'tab': "\t", 'newline': "\n", 'space': ' ', 'excl': '!', 'quot': '"', 'num': '#',
  'dollar': '$', 'percnt': '%', 'amp': '&', 'apos': "'", 'lpar': '(', 'rpar': ')',
  'ast': '*', 'plus': '+', 'comma': ',', 'minus': '-', 'period': '.', 'sol': '/',
  'colon': ':', 'semi': ';', 'lt': '<', 'equals': '=', 'gt': '>', 'quest': '?',
  'commat': '@', 'lbrack': '[', 'bsol': '\\', 'rbrack': ']', 'hat': '^', 'lowbar': '_',
  'grave': '`', 'lbrace': '{', 'vert': '|', 'rbrace': '}', 'tilde': '~'
}

precedence = (
  ('left', 'OR'),
  ('left', 'AND'),
  ('right', 'NOT'),
  ('left', 'PLUS', 'MINUS')
)


def p_statement_list1(p):
  '''statement_list : statement_list NEWLINE statement'''
  p[0] = p[1]
  p[0].extend(p[3])


def p_statement_list2(p):
  '''statement_list : statement_list compound_statement
                    | statement_list bfcode_statement'''
  p[0] = p[1]
  p[0].extend(p[2])


def p_statement_list3(p):
  '''statement_list : statement'''
  p[0] = Node('list', p[1])


def p_statement_list4(p):
  '''statement_list : '''
  p[0] = Node('list')


def p_statement1(p):
  '''statement : compound_statement
               | extended_statement
               | if_statement
               | while_statement
               | assign_statement
               | print_statement
               | bfcode_statement'''
  p[0] = p[1]


def p_compound_statement(p):
  '''compound_statement : LBRACK statement_list RBRACK'''
  p[0] =p[2]


def p_extended_statement(p):
  '''extended_statement : compound_statement bfcode'''
  p[0] = Node('list', p[1], p[2])


def p_if_statement1(p):
  '''if_statement : IF LPAREN expr RPAREN statement'''
  p[0] = Node('if', p[3], p[5], None)


def p_if_statement2(p):
  '''if_statement : IF LPAREN expr RPAREN statement ELSE statement'''
  p[0] = Node('if', p[3], p[5], p[7])


def p_while_statement(p):
  '''while_statement : WHILE LPAREN expr RPAREN statement'''
  p[0] = Node('while', p[3], p[5])


def p_assign_statement(p):
  '''assign_statement : variable EQUALS expr_list'''
  p[0] = Node('assign', p[1], p[3])


def p_print_statement(p):
  '''print_statement : expr_list'''
  p[0] = Node('print', p[1])


def p_bfcode_statement(p):
  '''bfcode_statement : bfcode'''
  p[0] = p[1]


def p_variable(p):
  '''variable : ID'''
  if p[1] in constants:
    pass
  else:
    p[0] = p[1]


def p_expr_list1(p):
  '''expr_list : expr_list expr'''<commit_msg>Update bfalparse.py<commit_after>from ply import *
import bfallex
import sys

class Node:
  def __init__(self, type, *values):
    self.type = type
    self.values = values
    self.count = len(values)

  def __repr__(self):
    return '%s%r'%(self.type, self.values)

  def extend(self, *values):
    self.values += values
    self.count += len(values)

tokens = bfallex.tokens

constants = {
  'tab': "\t", 'newline': "\n", 'space': ' ', 'excl': '!', 'quot': '"', 'num': '#',
  'dollar': '$', 'percnt': '%', 'amp': '&', 'apos': "'", 'lpar': '(', 'rpar': ')',
  'ast': '*', 'plus': '+', 'comma': ',', 'minus': '-', 'period': '.', 'sol': '/',
  'colon': ':', 'semi': ';', 'lt': '<', 'equals': '=', 'gt': '>', 'quest': '?',
  'commat': '@', 'lbrack': '[', 'bsol': '\\', 'rbrack': ']', 'hat': '^', 'lowbar': '_',
  'grave': '`', 'lbrace': '{', 'vert': '|', 'rbrace': '}', 'tilde': '~'
}

precedence = (
  ('left', 'OR'),
  ('left', 'AND'),
  ('right', 'NOT'),
  ('left', 'PLUS', 'MINUS')
)


def p_statement_list1(p):
  '''statement_list : statement_list NEWLINE statement'''
  p[0] = p[1]
  p[0].extend(p[3])


def p_statement_list2(p):
  '''statement_list : statement_list compound_statement
                    | statement_list bfcode_statement'''
  p[0] = p[1]
  p[0].extend(p[2])


def p_statement_list3(p):
  '''statement_list : statement'''
  p[0] = Node('list', p[1])


def p_statement_list4(p):
  '''statement_list : '''
  p[0] = Node('list')


def p_statement1(p):
  '''statement : compound_statement
               | extended_statement
               | if_statement
               | while_statement
               | assign_statement
               | print_statement
               | bfcode_statement'''
  p[0] = p[1]


def p_compound_statement(p):
  '''compound_statement : LBRACK statement_list RBRACK'''
  p[0] =p[2]


def p_extended_statement(p):
  '''extended_statement : compound_statement bfcode'''
  p[0] = Node('list', p[1], p[2])


def p_if_statement1(p):
  '''if_statement : IF LPAREN expr RPAREN statement'''
  p[0] = Node('if', p[3], p[5], None)


def p_if_statement2(p):
  '''if_statement : IF LPAREN expr RPAREN statement ELSE statement'''
  p[0] = Node('if', p[3], p[5], p[7])


def p_while_statement(p):
  '''while_statement : WHILE LPAREN expr RPAREN statement'''
  p[0] = Node('while', p[3], p[5])


def p_assign_statement(p):
  '''assign_statement : variable EQUALS expr_list'''
  p[0] = Node('assign', p[1], p[3])


def p_print_statement(p):
  '''print_statement : expr_list'''
  p[0] = Node('print', p[1])


def p_bfcode_statement(p):
  '''bfcode_statement : bfcode'''
  p[0] = p[1]


def p_variable(p):
  '''variable : ID'''
  if p[1] in constants:
    sys.stderr.write('Illegal assignment to constant %r on line %d\n' % (p[1], p.lineno(1)))
    sys.exit(1)
  else:
    p[0] = p[1]


def p_expr_list1(p):
  '''expr_list : expr_list expr'''
{'fraction_non_alphanumeric': 0.22790202342917998, 'fraction_numerical': 0.01686190983315584, 'mean_word_length': 3.699749791492911, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1727756', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2159, 'n_words': 452}