<commit_before>#!/usr/bin/env python
from pyparsing import Word, Literal, alphas, alphanums, Combine, Optional,\
                      ZeroOrMore, Forward, restOfLine, nums, lineno, col, \
                      ParseException
from pypepa.parsing.pepa_ast import *
from pypepa.parsing.rate_parser import RateParser
from pypepa.logger import init_log
from pypepa.exceptions import VariableNotDefinedError, VariableAlreadyDefinedError, \
                               ProcessNotDefinedError, ProcessNotDefinedError, \
                               ProcessAlreadyDefinedError

class PEPAParser(object):

    def __init__(self):
        self.log_pa = init_log()
        self._processes = {}
        self._var_stack = {}
        self._actions = []
        self._seen = {}
        self.systemeq = None
        self.rate_parser = RateParser()

    def _create_activity(self, string, loc,tok):
        self.log_pa.debug("Token: "+tok[0])
        n = ActivityNode("(" + tok[0] + "," + tok[1] + ")")
        n.action = tok[0]
        self._actions.append(n.action)
        n.rate = tok[1]
        return n

    def _create_procdef(self, s, loc, toks):
        self.log_pa.debug("Token: "+toks[0])
        n = ProcdefNode(toks[0])
        self._seen[toks[0]] = (loc, s)
        if len(toks) > 1:
            n.aggregation = True
            if toks[1] in self._var_stack:
                n.aggr_num = int(self._var_stack[toks[1]])
            else:
                n.aggr_num = int(toks[1])
        n.name = toks[0]
        return n

    def _create_definition(self, string, loc, tok):
        self.log_pa.debug("Left token: "+tok[0].data)
        self.log_pa.debug("Right token: "+tok[2].data)
        n = DefNode("=")
        n.left = tok[0]
        n.right = tok[2]
        n.process = tok[0].data
        if n.process in self._processes:
            message = "Process "+n.process+" already defined"
            self.log_pa.error(message)
            raise ProcessAlreadyDefinedError(message)
        self._processes[n.process] = n
        return n

    def _create_prefix(self,string, loc, tok):
        self.log_pa.debug("Tokens: "+str( len(tok) ))
        if len(tok) > 1:
            self.log_pa.debug("Left token: "+tok[0].data)
            self.log_pa.debug("Right token: "+tok[2].data)
            n = PrefixNode(".")
            lhs = tok[0]
            rhs = tok[2]
            n.left = lhs
            n.right = rhs
            return n
        else:
            self.log_pa.debug("Token: "+tok[0].data)
            return tok[0]

    def _create_choice(self,string,loc,tok):
        self.log_pa.debug("Tokens: "+str( len(tok) ))
        if not tok[0] is None:
            if len(tok) <3:
                self.log_pa.debug("Token: "+tok[0].data)
                return tok[0]
            else:
                self.log_pa.debug("Left token: "+tok[0].data)
                self.log_pa.debug("Right token: "+tok[2].data)

                n = ChoiceNode("+")
                n.left = tok[0]
                n.right = tok[2]
                return n

    def _create_coop(self,string, loc, tok):
        if not tok[0] is None:
            if len(tok) <3:
                self.log_pa.debug("Token: "+tok[0].data)
                return tok[0]
            else:
                self.log_pa.debug("Number of tokens" + str(len(tok)))
                self.log_pa.debug("Left token: "+tok[0].data)
                if tok[1].actionset is not None:
                    n = CoopNode("<"+str(tok[1].actionset)+">")
                    n.cooptype = "sync"
                    n.actionset = tok[1].actionset
                    self.log_pa.debug(n.actionset)
                else:
                    n = CoopNode("||")
                    n.cooptype = "par"
                if type(tok[2]).__name__ == "str":
                    self.log_pa.debug("String: "+tok[2])
                else:<commit_msg>Fixed * import from pepa_ast<commit_after>#!/usr/bin/env python
from pyparsing import Word, Literal, alphas, alphanums, Combine, Optional,\
                      ZeroOrMore, Forward, restOfLine, nums, lineno, col, \
                      ParseException
from pypepa.parsing.pepa_ast import ActivityNode, ProcdefNode, DefNode, CoopNode, \
                                    PrefixNode, SyncsetNode
from pypepa.parsing.rate_parser import RateParser
from pypepa.logger import init_log
from pypepa.exceptions import VariableNotDefinedError, VariableAlreadyDefinedError, \
                               ProcessNotDefinedError, ProcessNotDefinedError, \
                               ProcessAlreadyDefinedError

class PEPAParser(object):

    def __init__(self):
        self.log_pa = init_log()
        self._processes = {}
        self._var_stack = {}
        self._actions = []
        self._seen = {}
        self.systemeq = None
        self.rate_parser = RateParser()

    def _create_activity(self, string, loc,tok):
        self.log_pa.debug("Token: "+tok[0])
        n = ActivityNode("(" + tok[0] + "," + tok[1] + ")")
        n.action = tok[0]
        self._actions.append(n.action)
        n.rate = tok[1]
        return n

    def _create_procdef(self, s, loc, toks):
        self.log_pa.debug("Token: "+toks[0])
        n = ProcdefNode(toks[0])
        self._seen[toks[0]] = (loc, s)
        if len(toks) > 1:
            n.aggregation = True
            if toks[1] in self._var_stack:
                n.aggr_num = int(self._var_stack[toks[1]])
            else:
                n.aggr_num = int(toks[1])
        n.name = toks[0]
        return n

    def _create_definition(self, string, loc, tok):
        self.log_pa.debug("Left token: "+tok[0].data)
        self.log_pa.debug("Right token: "+tok[2].data)
        n = DefNode("=")
        n.left = tok[0]
        n.right = tok[2]
        n.process = tok[0].data
        if n.process in self._processes:
            message = "Process "+n.process+" already defined"
            self.log_pa.error(message)
            raise ProcessAlreadyDefinedError(message)
        self._processes[n.process] = n
        return n

    def _create_prefix(self,string, loc, tok):
        self.log_pa.debug("Tokens: " + str(len(tok)))
        if len(tok) > 1:
            self.log_pa.debug("Left token: " + tok[0].data)
            self.log_pa.debug("Right token: "+ tok[2].data)
            n = PrefixNode(".")
            n.left = tok[0]
            n.right = tok[2]
            return n
        else:
            self.log_pa.debug("Token: " + tok[0].data)
            return tok[0]

    def _create_choice(self,string,loc,tok):
        self.log_pa.debug("Tokens: " + str(len(tok)))
        if not tok[0] is None:
            if len(tok) <3:
                self.log_pa.debug("Token: "+ tok[0].data)
                return tok[0]
            else:
                self.log_pa.debug("Left token: " + tok[0].data)
                self.log_pa.debug("Right token: " + tok[2].data)
                n = ChoiceNode("+")
                n.left = tok[0]
                n.right = tok[2]
                return n

    def _create_coop(self,string, loc, tok):
        if not tok[0] is None:
            if len(tok) <3:
                self.log_pa.debug("Token: "+tok[0].data)
                return tok[0]
            else:
                self.log_pa.debug("Number of tokens" + str(len(tok)))
                self.log_pa.debug("Left token: "+tok[0].data)
                if tok[1].actionset is not None:
                    n = CoopNode("<"+str(tok[1].actionset)+">")
                    n.cooptype = "sync"
                    n.actionset = tok[1].actionset
                    self.log_pa.debug(n.actionset)
                else:
                    n = CoopNode("||")
                    n.cooptype = "par"
                if type(tok[2]).__name__ == "str":
                    self.log_pa.debug("String: "+tok[2])
                else:
{'fraction_non_alphanumeric': 0.13024675655049606, 'fraction_numerical': 0.010938692444670568, 'mean_word_length': 1.6288866599799399, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4087126', 'n_tokens_mistral': 2535, 'n_tokens_neox': 2445, 'n_words': 492}