<commit_before>import logging
import sys
import string

try:
    import pyparsing
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList 
except ImportError:
    logging.critical("PyParsing have to be installed on the computer")
    sys.exit(-1)

from machine import Machine
from monoid import Monoid
from constants import deep_cases
from lexicon import Lexicon

class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])

    lb = "["
    rb = "]"
    lp = "("
    rp = ")"

    def_sep = ":"
    arg_sep = ","
    part_sep = ";"
    comment_sep = "%"
    prime = "'"
    hyphen = "-"

    def __init__(self):
        self.init_parser()
        self.lexicon = Lexicon()

    @classmethod
    def _is_binary(cls, s):
        return type(s) in cls._str and s.isupper() and not s in deep_cases
    
    @classmethod
    def _is_unary(cls, s):
        # TODO unaries can contain hyphens
        return type(s) in cls._str and s.islower() or s in deep_cases
    
    @classmethod
    def _is_deep_case(cls, s):
        return s in deep_cases
    
    def init_parser(self):
        self.lb_lit = Literal(DefinitionParser.lb)
        self.rb_lit = Literal(DefinitionParser.rb)
        self.lp_lit = Literal(DefinitionParser.lp)
        self.rp_lit = Literal(DefinitionParser.rp)

        self.def_sep_lit = Literal(DefinitionParser.def_sep)
        self.arg_sep_lit = Literal(DefinitionParser.arg_sep)
        self.part_sep_lit = Literal(DefinitionParser.part_sep)
        self.comment_sep_lit = Literal(DefinitionParser.comment_sep)
        self.prime_lit = Literal(DefinitionParser.prime)
        self.hyphen_lit = Literal(DefinitionParser.hyphen)
        
        self.deep_cases = reduce(lambda a, b: a | b,
            (Literal(dc) for dc in deep_cases))
        
        self.unary = Combine(Optional("-") + Word(string.lowercase + "_") +
                             Optional(Word(nums))) | self.deep_cases
        self.binary = Word(string.uppercase + "_" + nums)
        self.dontcare = SkipTo(LineEnd())
        
        # main expression
        self.expression = Forward()
        
        # "enumerable expression"
        # D -> E | E, D
        self.definition = Group(delimitedList(self.expression,
            delim=DefinitionParser.arg_sep))
        self.expression << Group(
            # E -> U
            (self.unary) ^

            # E -> U [ D ]
            (self.unary + self.lb_lit.suppress() + self.definition + self.rb_lit.suppress() ) ^ 

            # E -> U ( U ) | U ( U [ E ] )
            (self.unary + self.lp_lit + self.unary + Optional(self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) + self.rp_lit ) ^

            # E -> U B
            (self.unary + self.binary) ^
            
            # E -> U B E
            (self.unary + self.binary + self.expression) ^

            # E -> B E
            (self.binary + self.expression) ^

            # E -> B [ E ]
            (self.binary + self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) ^
            
            # E -> B [ E ; E ] 
            (self.binary + self.lb_lit.suppress() + self.expression + self.part_sep_lit.suppress() + self.expression + self.rb_lit.suppress()) ^
            
            # E -> [ E ] B
            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary) ^
            
            # E -> [ E ] B [ E ]
            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary + self.lb_lit.suppress() + self.expression + self.rb_lit.suppress()) ^
            
            # E -> [ E ] B E
            (self.lb_lit.suppress() + self.expression + self.rb_lit.suppress() + self.binary + self.expression ) ^
            
            # E -> 'B
            (self.prime_lit + self.binary) ^

            # E -> B'
            (self.binary + self.prime_lit)
        )
        
        self.hu, self.pos, self.en, self.lt, self.pt = (Word(alphanums + "#-/_" ),) * 5
        self.defid = Word(nums)
        self.word = Group(self.hu + self.pos + self.en + self.lt + self.pt)

        # S -> W : D | W : D % _
        self.sen = (self.defid + self.word + self.def_sep_lit.suppress() + Optional(self.definition) + Optional(self.comment_sep_lit + self.dontcare).suppress()) + LineEnd()
    
    def parse(self, s):
        return self.sen.parseString(s).asList()

    def get_machine(self, printname):
        if printname not in self.lexicon.static:
            part_num = 0
            if DefinitionParser._is_unary(printname):
                part_num = 1
            elif DefinitionParser._is_binary(printname):
                part_num = 2
            if part_num == 0:
                raise ValueError("get_machine() is called with an invalid printname argument")
            logging.debug("Creating machine {0} with {1} partitions".format(printname, part_num))
            self.lexicon.add_static(Machine(Monoid(printname, part_num)))
        return self.lexicon.static[printname]
    
    def __parse_expr(self, expr, parent):
        """
        creates machines from a parse node and its children
        there should be one handler for every rule
        """

        logging.debug("Parsing expression: {0}".format(expr))

        # name shortening for classmethods
        cls = DefinitionParser

        is_binary = cls._is_binary
        is_unary = cls._is_unary
        is_tree = lambda r: type(r) == list

        # E -> U [ D ]
        # ['unary', [['unary'], ['unary']]]
        if (len(expr) == 2 and
              is_unary(expr[0]) and
              is_tree(expr[1])):
            m = self.get_machine(expr[0])
            for _property in expr[1]:
                m.append(self.__parse_expr(_property, m), 1)
            return m

        # E -> U ( U ) | U ( U [ E ] )
        # ['unary', '(', 'unary', ')']
        if (len(expr) == 4 and
              is_unary(expr[0]) and
              expr[1] == cls.lp and
              is_unary(expr[2]) and
              expr[3] == cls.rp):
            m = self.get_machine(expr[2])
            m.append(self.get_machine(expr[0]), 1)
            return m

        # E -> U B E
        # ['unary', 'BINARY', ['unary']]
        if (len(expr) == 3 and
              is_unary(expr[0]) and
              is_binary(expr[1]) and
              is_tree(expr[2])):
            m = self.get_machine(expr[1])
            m.append(self.get_machine(expr[0]), 1)
            m.append(self.__parse_expr(expr[2], m), 2)
            return m

        # E -> U B
        # ['unary', 'BINARY']
        if (len(expr) == 2 and
              is_unary(expr[0]) and
              is_binary(expr[1])):
            m = self.get_machine(expr[1])
            m.append(self.get_machine(expr[0]), 1)
            return m

        # E -> U
        # ['unary']
        if (len(expr) == 1 and
              is_unary(expr[0])):
            return self.get_machine(expr[0])

        # E -> B [ E ; E ] 
        # ['BINARY', ['unary'], ['unary']]
        if (len(expr) == 3 and
              is_binary(expr[0]) and
              is_tree(expr[1]) and
              is_tree(expr[2])):
            m = self.get_machine(expr[0])
            m.append(self.__parse_expr(expr[1], m), 1)
            m.append(self.__parse_expr(expr[2], m), 2)
            return m

        # E -> [ E ] B [ E ]
        # E -> [ E ] B E # TODO test whether this is working too?
        # [['unary'], 'BINARY', ['unary']]
        if (len(expr) == 3 and
              is_tree(expr[0]) and
              is_binary(expr[1]) and
              is_tree(expr[2])):
            m = self.get_machine(expr[1])
            m.append(self.__parse_expr(expr[0], m), 1)
            m.append(self.__parse_expr(expr[2], m), 2)
            return m

        # E -> B [ E ]
        # ['BINARY', ['unary']]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            is_tree(expr[1])):
            m = self.get_machine(expr[0])
            m.append(self.__parse_expr(expr[1], m), 2)
            return m

        # E -> [ E ] B
        # [['unary'], 'BINARY']
        if (len(expr) == 2 and
            is_tree(expr[0]) and
            is_binary(expr[1])):
            m = self.get_machine(expr[1])
            m.append(self.__parse_expr(expr[0], m), 1)
            return m

        # E -> B E
        # ['BINARY', ['unary']]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            is_tree(expr[1])):
            m = self.get_machine(expr[0])
            m.append(self.__parse_expr(expr[1], m), 2)
            return m

        # E -> 'B
        # ["'", 'BINARY']
        if (len(expr) == 2 and
            expr[0] == cls.prime and
            is_binary(expr[1])):
            m = self.get_machine(expr[1])
            m.append(parent, 2)
            return None

        # E -> B'
        # ['BINARY', "'"]
        if (len(expr) == 2 and
            is_binary(expr[0]) and
            expr[1] == cls.prime):
            m = self.get_machine(expr[0])
            m.append(parent, 1)
            return None
        
        pe = ParserException("Unknown expression in definition")
        logging.debug(str(pe))
        logging.debug(expr)
        raise pe
    
    def parse_into_machines(self, s):
        parsed = self.parse(s)
        
        # HACK printname is now set to english
        machine = self.get_machine(parsed[1][3])
        if len(parsed) > 2:
            for d in parsed[2]:
                machine.append(self.__parse_expr(d, machine), 1)

def read(f):
    dp = DefinitionParser()
    for line in f:
        l = line.strip()
        logging.info("Parsing: {0}".format(l))
        if len(l) == 0:
            continue
        if l.startswith("#"):
            continue<commit_msg>modified:   definition_parser.py new grammar implemented that is written by MaM and supposed to be free of recursive rules and easier to understand, but not really tested yet<commit_after>import logging
import sys
import string

try:
    import pyparsing
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList 
except ImportError:
    logging.critical("PyParsing have to be installed on the computer")
    sys.exit(-1)

from machine import Machine
from monoid import Monoid
from constants import deep_cases

class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])

    lb = "["
    rb = "]"
    lp = "("
    rp = ")"

    def_sep = ":"
    arg_sep = ","
    part_sep = ";"
    comment_sep = "%"
    prime = "'"
    hyphen = "-"

    def __init__(self):
        self.init_parser()

    @classmethod
    def _is_binary(cls, s):
        return type(s) in cls._str and s.isupper() and not s in deep_cases
    
    @classmethod
    def _is_unary(cls, s):
        # TODO unaries can contain hyphens
        return type(s) in cls._str and s.islower() or s in deep_cases
    
    @classmethod
    def _is_deep_case(cls, s):
        return s in deep_cases

    def init_parser(self):
        self.lb_lit = Literal(DefinitionParser.lb)
        self.rb_lit = Literal(DefinitionParser.rb)
        self.lp_lit = Literal(DefinitionParser.lp)
        self.rp_lit = Literal(DefinitionParser.rp)

        self.def_sep_lit = Literal(DefinitionParser.def_sep)
        self.arg_sep_lit = Literal(DefinitionParser.arg_sep)
        self.part_sep_lit = Literal(DefinitionParser.part_sep)
        self.comment_sep_lit = Literal(DefinitionParser.comment_sep)
        self.prime_lit = Literal(DefinitionParser.prime)
        self.hyphen_lit = Literal(DefinitionParser.hyphen)
        
        self.deep_cases = reduce(lambda a, b: a | b,
            (Literal(dc) for dc in deep_cases))
        
        self.unary = Combine(Optional("-") + Word(string.lowercase + "_") +
                             Optional(Word(nums))) | self.deep_cases
        self.binary = Word(string.uppercase + "_" + nums)
        self.dontcare = SkipTo(LineEnd())
        
        # main expression
        self.expression = Forward()
        self.binexpr = Forward()
        self.unexpr = Forward()
        self.argexpr = Forward()
        
        # "enumerable expression"
        # D -> E | E, D
        self.definition = Group(delimitedList(self.expression,
            delim=DefinitionParser.arg_sep))

        self.expression << Group(
            # E -> UE
            (self.unexpr) ^

            # E -> BE
            (self.binexpr) ^

            # E -> U ( BE )
            (self.unexpr + self.lp_lit + self.binexpr + self.rp_lit)
        )

        self.binexpr << Group(
            # BE -> A B A
            (self.argexpr + self.binary + self.argexpr) ^

            # BE -> A B
            (self.argexpr + self.binary) ^

            # BE -> B A
            (self.binary + self.argexpr) ^

            # BE -> B [ E; E ]
            (self.binary + self.lb_lit + self.expression + self.part_sep_lit + self.expression + self.rb_lit) ^

            # BE -> 'B
            (self.prime_lit + self.binary) ^

            # BE -> B'
            (self.binary + self.prime_lit)
        )

        self.unexpr << Group(
            # UE -> U
            (self.unary) ^

            # UE -> U [ D ]
            (self.unary + self.lb_lit + self.definition + self.rb_lit) ^

            # UE -> U ( U )
            (self.unary + self.lp_lit + self.unary + self.rp_lit)
        )

        self.argexpr << Group(
            # A -> UE
            (self.unexpr) ^

            # A -> [ D ]
            (self.lb_lit + self.definition + self.rb_lit)
        )
        
        self.hu, self.pos, self.en, self.lt, self.pt = (Word(alphanums + "#-/_" ),) * 5
        self.defid = Word(nums)
        self.word = Group(self.hu + self.pos + self.en + self.lt + self.pt)

        # S -> W : D | W : D % _
        self.sen = (self.defid + self.word + self.def_sep_lit.suppress() + Optional(self.definition) + Optional(self.comment_sep_lit + self.dontcare).suppress()) + LineEnd()
    
    def parse(self, s):
        return self.sen.parseString(s).asList()

    def __parse_expr(self, expr, parent):
        """
        creates machines from a parse node and its children
        there should be one handler for every rule
        """

        logging.debug("Parsing expression: {0}".format(expr))

        # name shortening for classmethods
        cls = DefinitionParser

        is_binary = cls._is_binary
        is_unary = cls._is_unary
        is_tree = lambda r: type(r) == list

        if (len(expr) == 1):
            # E -> UE | BE
            # A -> UE
            if (is_tree(expr[0])):
                return self.__parse_expr(expr[0], parent)

            # UE -> U
            if (is_unary(expr[0])):
                return Machine(Monoid(expr[0], 1))

        if (len(expr) == 2):
            # BE -> A B
            if (is_tree(expr[0]) and
                    is_binary(expr[1])):
                m = Machine(Monoid(expr[1], 2))
                m.append(self.__parse_expr(expr[0], m), 1)
                return m

            # BE -> B A
            if (is_binary(expr[0]) and
                    is_tree(expr[1])):
                m = Machine(Monoid(expr[0], 2))
                m.append(self.__parse_expr(expr[1], m), 2)
                return m

            # BE -> 'B
            if (expr[0] == "'" and
                    is_binary(expr[1])):
                m = Machine(Monoid(expr[1], 2))
                m.append(parent, 2)
                # nothing to append to any partitions
                return None

            # BE -> B'
            if (is_binary(expr[0]) and
                    expr[1] == "'"):
                m = Machine(Monoid(expr[0], 2))
                m.append(parent, 1)
                # nothing to append to any partitions
                return None

        if (len(expr) == 3):
            # UB -> A B A
            if (is_tree(expr[0]) and
                    is_binary(expr[1]) and
                    is_tree(expr[2])):
                m = Machine(Monoid(expr[1], 2))
                m.append(self.__parse_expr(expr[0], m), 1)
                m.append(self.__parse_expr(expr[2], m), 2)
                return m

            # A -> [ D ]
            if (expr[0] == "[" and
                    is_tree(expr[1]) and
                    expr[2] == "]"):
                return list(self.__parse_definition(expr[1], parent))
        
        if (len(expr) == 4):
            # E -> U ( BE )
            if (is_unary(expr[0]) and
                    expr[1] == "(" and
                    is_tree(expr[2]) and
                    expr[3] == ")"):
                m = self.__parse_expr(expr[2], parent)
                m.append(Machine(Monoid(expr[1], 1)), 0)
                return m

            # UE -> U [ D ]
            if (is_unary(expr[0]) and
                    expr[1] == "[" and
                    is_tree(expr[2]) and
                    expr[3] == "]"):
                m = Machine(Monoid(expr[0], 1))
                for parsed_expr in self.__parse_definition(expr[2], m):
                    m.append(parsed_expr, 1)
                return m

            # UE -> U ( U )
            if (is_unary(expr[0]) and
                    expr[1] == "(" and
                    is_unary(expr[2]) and
                    expr[3] == ")"):
                m = Machine(Monoid(expr[2], 1))
                m.append(Machine(Monoid(expr[0], 1)), 1)
                return m

        if (len(expr) == 6):
            # BE -> B [E; E]
            if (is_binary(expr[0]) and
                    expr[1] == "[" and
                    is_tree(expr[2]) and
                    expr[3] == ";" and
                    is_tree(expr[4]) and
                    expr[5] == "]"):
                m = Machine(Monoid(expr[0], 2))
                m.append(self.__parse_expr(expr[2], m), 1)
                m.append(self.__parse_expr(expr[4], m), 2)
                return m

        pe = ParserException("Unknown expression in definition")
        logging.debug(str(pe))
        logging.debug(expr)
        raise pe

    def __parse_definition(self, definition, parent):
        for d in definition:
            yield self.__parse_expr(d, parent)
    
    def parse_into_machines(self, s):
        parsed = self.parse(s)
        
        # HACK printname is now set to english
        machine = Machine(Monoid(parsed[1][2], 1))
        if len(parsed) > 2:
            for parsed_expr in self.__parse_definition(parsed[2], machine):
                machine.append(parsed_expr, 1)
        return machine

def read(f):
    dp = DefinitionParser()
    for line in f:
        l = line.strip()
        logging.info("Parsing: {0}".format(l))
        if len(l) == 0:
            continue
        if l.startswith("#"):
            continue
{'fraction_non_alphanumeric': 0.13043016463090812, 'fraction_numerical': 0.009930961232076473, 'mean_word_length': 1.5852553542009884, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272337', 'n_tokens_mistral': 6032, 'n_tokens_neox': 5821, 'n_words': 1323}