<commit_before>#!/usr/bin/python
#
# Author : hdj <<EMAIL>>
# Date   : 2014.06
# Project: maaps
# Description:
#   TODO: add some description!
#
import os
import copy
import ply.lex as lex
import ply.yacc as yacc
from templating import Templating
from re import IGNORECASE


reserved = {
    'if'    : 'IF',
    'then'  : 'THEN',
    'else'  : 'ELSE',
    'while' : 'WHILE',
    'python': 'PYTHON',
}


# list of tokens (always required)
tokens = [
    'IMPORT',
    'CHAIN',
    'ENTRYPOINT',
    'MODULE',
    'CALL',
    'LOOP',
    'ONEXCEPTION',

    'LBRACE',
    'RBRACE',

    'IDENTIFER',
    'COMMENT',
    'EQUAL',
    'STRING',
    'NUMBER',
    'CODE',
] + list(reserved.values())

states = (
     ('code', 'exclusive'),
)

t_EQUAL         = r'='
t_LBRACE        = r'\{'
t_RBRACE        = r'\}'
t_ignore        = r' \t'

def t_COMMENT(t):
    r'\#.*'
    pass

# =================================================================
# == START OF Code|Context Section
# =================================================================
def t_CODE(t):
    r'(CODE|CONTEXT)\s+=\s+\{'
    t.lexer.code_start = t.lexer.lexpos        # Record the starting position
    t.lexer.level = 1                          # Initial brace level
    t.lexer.begin('code')                     # Enter 'ccode' state

def t_code_LBRACE(t):
    r'\{'
    t.lexer.level += 1

def t_code_RBRACE(t):
    r'\}'
    t.lexer.level -= 1

    # If closing brace, return the code fragment
    if t.lexer.level == 0:
        t.value = t.lexer.lexdata[t.lexer.code_start:t.lexer.lexpos-1]
        t.type = "CODE"
        t.lexer.lineno += t.value.count('\n')
        t.lexer.begin('INITIAL')
        return t

# Any sequence of non-whitespace characters (not braces, strings)
def t_code_nonspace(t):
    r'[^\s\{\}\'\"]+'

# Ignored characters (whitespace)
t_code_ignore = " \t\n"

# For bad characters, we just skip over it
def t_code_error(t):
    t.lexer.skip(1)
# =================================================================
# == END OF Code|Context Section
# =================================================================
#
# this must stay under CODE section for propper parsing/identifying.
def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t
def t_STRING(t):
    r'[\'"]([^\\\n]|(\\.))*?[\'"]'
    #r'\"([^\\\n]|(\\.))*?\"'
    #t.value = t.value.lstrip('"').rstrip('"')
    return t

def t_MODULE(t):
    r'MODULE\s'
    t.value = t.value.upper().strip()<commit_msg>fix bug for expressions starting with a 't' the 't' is ignored. (grrrrrr)<commit_after>#!/usr/bin/python
#
# Author : hdj <<EMAIL>>
# Date   : 2014.06
# Project: maaps
# Description:
#   TODO: add some description!
#
import os
import copy
import ply.lex as lex
import ply.yacc as yacc
from templating import Templating
from re import IGNORECASE


reserved = {
    'if'    : 'IF',
    'then'  : 'THEN',
    'else'  : 'ELSE',
    'while' : 'WHILE',
    'python': 'PYTHON',
}


# list of tokens (always required)
tokens = [
    'IMPORT',
    'CHAIN',
    'ENTRYPOINT',
    'MODULE',
    'CALL',
    'LOOP',
    'ONEXCEPTION',

    'LBRACE',
    'RBRACE',

    'IDENTIFER',
    'COMMENT',
    'EQUAL',
    'STRING',
    'NUMBER',
    'CODE',
] + list(reserved.values())

states = (
     ('code', 'exclusive'),
)

t_EQUAL         = r'='
t_LBRACE        = r'\{'
t_RBRACE        = r'\}'
t_ignore        = ' \t'

def t_COMMENT(t):
    r'\#.*'
    pass

# =================================================================
# == START OF Code|Context Section
# =================================================================
def t_CODE(t):
    r'(CODE|CONTEXT)\s+=\s+\{'
    t.lexer.code_start = t.lexer.lexpos        # Record the starting position
    t.lexer.level = 1                          # Initial brace level
    t.lexer.begin('code')                     # Enter 'ccode' state

def t_code_LBRACE(t):
    r'\{'
    t.lexer.level += 1

def t_code_RBRACE(t):
    r'\}'
    t.lexer.level -= 1

    # If closing brace, return the code fragment
    if t.lexer.level == 0:
        t.value = t.lexer.lexdata[t.lexer.code_start:t.lexer.lexpos-1]
        t.type = "CODE"
        t.lexer.lineno += t.value.count('\n')
        t.lexer.begin('INITIAL')
        return t

# Any sequence of non-whitespace characters (not braces, strings)
def t_code_nonspace(t):
    r'[^\s\{\}\'\"]+'

# Ignored characters (whitespace)
t_code_ignore = " \t\n"

# For bad characters, we just skip over it
def t_code_error(t):
    t.lexer.skip(1)

# =================================================================
# == END OF Code|Context Section
# =================================================================
#
# this must stay under CODE section for propper parsing/identifying.
def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)
    return t
def t_STRING(t):
    r'[\'"]([^\\\n]|(\\.))*?[\'"]'
    #r'\"([^\\\n]|(\\.))*?\"'
    #t.value = t.value.lstrip('"').rstrip('"')
    return t

def t_MODULE(t):
    r'MODULE\s'
    t.value = t.value.upper().strip()
{'fraction_non_alphanumeric': 0.26675977653631283, 'fraction_numerical': 0.004788507581803671, 'mean_word_length': 2.7216035634743876, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1016691', 'n_tokens_mistral': 1839, 'n_tokens_neox': 1771, 'n_words': 396}