<commit_before>from dolo.symbolic.symbolic import Variable, Shock, Parameter, Equation
from dolo.symbolic.model import Model

import sympy
import re
import inspect


line_regex = re.compile(
    "(\s*)$|"             # blank
    +"\s*@#(.*)$|"        # macro
    +"\s*//[^#](.*)$|"    # comment
    +"\s*%[^#](.*)$|"    # comment
    +"\s*\[(.*)\]\s*$|"   # tag
    +"(\s*[^\s].*)$"      # instruction
)
tag_regex = re.compile("\s*(\w+)\s*=\s*'(.*)'")

def parse_dynare_text(txt,add_model=True,full_output=False,names_dict = {}, debug=False):
    '''
    Imports the content of a modfile into the current interpreter scope
    '''
    # here we call "instruction group", a string finishing by a semicolon
    # an "instruction group" can have several lines
    # a line can be
    # - a comment //...
    # - an old-style tag //$...
    # - a new-style tag [key1='value1',..]
    # - macro-instruction @#...
    # A Modfile contains several blocks (in this order) :
    # - an initblock defining variables, exovariables, parameters, initialization
    #   inside the initblock the order of declaration doesn't matter
    # - a model block with two special lines (model; end;)
    # - optional blocks (like endval, shocks)
    #    seperated by free matlab instructions in any order;
    # - all other instructions are ignored

    otxt = txt
    otxt = otxt.replace("\r\n","\n")
    otxt = otxt.replace("^","**")

    # first, we remove end-of-line comments : they are definitely lost
    regex = re.compile("(.+)//[^#](.*)")
    def remove_end_comment(line):
        res = regex.search(line)
        if res:
            l = res.groups(1)[0]
            return(l)
        else:
            return line
    txt = str.join("\n",map(remove_end_comment,otxt.split("\n")))

    name_regex = re.compile("//\s*fname\s*=\s*'(.*)'")
    m = name_regex.search(txt)
    if m:
        fname = m.group(1)
    else:
        fname = None


    instruction_groups = [Instruction_group(s) for s in txt.split(";")]

    instructions = [ig.instruction for ig in instruction_groups]

    if debug:
        print('Elementary instructions')
        for i in instruction_groups:
            print(i)

    for ins in instructions:
        print(ins)
    try:
        imodel = [re.compile('model(\(.*\)|)').match(e) is not None for e in instructions]
        imodel = imodel.index(True)
        #imodel = instructions.index("model") #this doesn't work for "MODEL"
        iend = instructions.index("end")
        model_block = instruction_groups[imodel:(iend+1)]
        init_block = instruction_groups[0:imodel]
    except:
        raise Exception('Model block could not be found.')

    next_instructions = instructions[(iend+1):]<commit_msg>BUG: Suppressed unnecessary print statement.<commit_after>from dolo.symbolic.symbolic import Variable, Shock, Parameter, Equation
from dolo.symbolic.model import Model

import sympy
import re
import inspect


line_regex = re.compile(
    "(\s*)$|"             # blank
    +"\s*@#(.*)$|"        # macro
    +"\s*//[^#](.*)$|"    # comment
    +"\s*%[^#](.*)$|"    # comment
    +"\s*\[(.*)\]\s*$|"   # tag
    +"(\s*[^\s].*)$"      # instruction
)
tag_regex = re.compile("\s*(\w+)\s*=\s*'(.*)'")

def parse_dynare_text(txt,add_model=True,full_output=False,names_dict = {}, debug=False):
    '''
    Imports the content of a modfile into the current interpreter scope
    '''
    # here we call "instruction group", a string finishing by a semicolon
    # an "instruction group" can have several lines
    # a line can be
    # - a comment //...
    # - an old-style tag //$...
    # - a new-style tag [key1='value1',..]
    # - macro-instruction @#...
    # A Modfile contains several blocks (in this order) :
    # - an initblock defining variables, exovariables, parameters, initialization
    #   inside the initblock the order of declaration doesn't matter
    # - a model block with two special lines (model; end;)
    # - optional blocks (like endval, shocks)
    #    seperated by free matlab instructions in any order;
    # - all other instructions are ignored

    otxt = txt
    otxt = otxt.replace("\r\n","\n")
    otxt = otxt.replace("^","**")

    # first, we remove end-of-line comments : they are definitely lost
    regex = re.compile("(.+)//[^#](.*)")
    def remove_end_comment(line):
        res = regex.search(line)
        if res:
            l = res.groups(1)[0]
            return(l)
        else:
            return line
    txt = str.join("\n",map(remove_end_comment,otxt.split("\n")))

    name_regex = re.compile("//\s*fname\s*=\s*'(.*)'")
    m = name_regex.search(txt)
    if m:
        fname = m.group(1)
    else:
        fname = None


    instruction_groups = [Instruction_group(s) for s in txt.split(";")]

    instructions = [ig.instruction for ig in instruction_groups]

    if debug:
        print('Elementary instructions')
        for i in instruction_groups:
            print(i)

    try:
        imodel = [re.compile('model(\(.*\)|)').match(e) is not None for e in instructions]
        imodel = imodel.index(True)
        #imodel = instructions.index("model") #this doesn't work for "MODEL"
        iend = instructions.index("end")
        model_block = instruction_groups[imodel:(iend+1)]
        init_block = instruction_groups[0:imodel]
    except:
        raise Exception('Model block could not be found.')

    next_instructions = instructions[(iend+1):]
{'fraction_non_alphanumeric': 0.15624423538092602, 'fraction_numerical': 0.002951484965873455, 'mean_word_length': 2.5600787918581744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5069723', 'n_tokens_mistral': 1781, 'n_tokens_neox': 1722, 'n_words': 464}