<commit_before># Copyright (c) 2010-2014 <NAME>
# Copyright (c) 2010-2014 <NAME>
# Copyright (c) 2010-2014 Stony Brook University
# Copyright (c) 2010-2014 The Research Foundation of SUNY
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import sys

from ast import *
from . import dast
from .pygen import *
from .parser import Pattern2Constant
from .utils import printe, printd, printw, OptionsManager, to_source

INC_MODULE_VAR = "IncModule"

QUERY_STUB_FORMAT = "Query_%d"
ASSIGN_STUB_FORMAT = "Assign_%s"
DEL_STUB_FORMAT = "Delete_%s"
RESET_STUB_FORMAT = "Reset_%s_Events"
INIT_STUB_PREFIX = "Init_"
UPDATE_STUB_PREFIX = "Update_"
LOCAL_WITNESS_SET = "__witness_set__"
LOCAL_RESULT_VAR = "__result__"
GLOBAL_WITNESS_VAR = "Witness"
SELF_ID_NAME = "SELF_ID"
JB_STYLE_MODULE = "invinc.runtime"
JB_STYLE_SET = JB_STYLE_MODULE + ".Set"
JB_STYLE_MAP = JB_STYLE_MODULE + ".Map"

NegatedOperatorMap = {
    dast.EqOp : NotEq,
    dast.NotEqOp : Eq,
    dast.LtOp : Gt,
    dast.LtEOp : GtE,
    dast.GtOp : Lt,
    dast.GtEOp : LtE,
    dast.IsOp : IsNot,
    dast.IsNotOp : Is,
    dast.InOp : NotIn,
    dast.NotInOp : In
}

##################################################
# Global states:

Options = None                  # Command line options
ModuleFilename = ""

##########
# Auxiliary methods:

def iprintd(message):
    printd(message, filename=ModuleFilename)

def iprintw(message):
    printw(message, filename=ModuleFilename)

def combine_not_comparison(node):
    if (isinstance(node, dast.LogicalExpr) and
            node.operator is dast.NotOp and
            isinstance(node.left, dast.ComparisonExpr)):
        exp = dast.ComparisonExpr(parent=node.parent, ast=node.ast)
        exp.left = node.left.left
        exp.right = node.left.right
        exp.left._parent = exp
        exp.right._parent = exp
        exp.comparator = NegatedOperatorMap[node.left.comparator]
        return exp
    else:
        return node

def apply_demorgan_rule(node):
    if (isinstance(node, dast.LogicalExpr) and
            node.operator is dast.NotOp and
            isinstance(node.left, dast.LogicalExpr) and
            node.left.operator in {dast.AndOp, dast.OrOp}):
        exp = dast.LogicalExpr(parent=node.parent, ast=node.ast,
                               op=NegatedOperatorMap[node.left.operator])
        exp.subexprs = [dast.LogicalExpr(parent=exp, ast=None,
                                         op=dast.NotOp,
                                         subexprs=[e])
                        for e in node.left.subexprs]
        for e in exp.subexprs:
            e.left._parent = e
        return exp
    else:
        return node

def domain_for_condition(domainspec, condition):
    expr = dast.SetCompExpr(domainspec.parent)
    expr.conditions.append(domainspec.clone())
    expr.conditions.append(condition)
    if len(domainspec.pattern.ordered_freevars) == 1:
        expr.elem = dast.SimpleExpr(expr)
        expr.elem.value = domainspec.pattern.ordered_freevars[0]
        domainspec.pattern = dast.PatternExpr(domainspec)
        domainspec.pattern.pattern = dast.FreePattern(
            domainspec.pattern, value=expr.elem.value)
    else:
        expr.elem = dast.TupleExpr(expr)
        expr.elem.subexprs = [dast.SimpleExpr(expr.elem, value=v)
                              for v in domainspec.pattern.ordered_freevars]
        domainspec.pattern = dast.PatternExpr(domainspec)
        domainspec.pattern.pattern = dast.TuplePattern(
            domainspec.pattern,
            value = [dast.FreePattern(domainspec.pattern, value=fv.value)
                     for fv in expr.elem.subexprs])
    domainspec.domain = expr
    iprintd("domain_for_condition: " + str(expr))
    return expr

def optimize_tuple(elt):
    """Expand single element tuples."""

    if type(elt) is Tuple and len(elt.elts) == 1:
        elt = elt.elts[0]
    return elt

def ast_eq(left, right):
    """Structural equality of AST nodes."""

    if type(left) is not type(right):
        return False
    if isinstance(left, list):
        if len(left) != len(right):
            return False
        for litem, ritem in zip(left, right):
            if not ast_eq(litem, ritem):
                return False
        return True
    if not isinstance(left, AST):
        return left == right
    for (left_fieldname, left_value), (right_fieldname, right_value) in \
            zip(iter_fields(left), iter_fields(right)):
        if not (left_fieldname == right_fieldname):
            break
        if not ast_eq(left_value, right_value):
            break
    else:
        return True
    return False

def is_all_wildcards(targets):
    """True if 'targets' contain only wildcards."""

    for elt in targets:
        if not (isinstance(elt, Name) and elt.id == '_'):
            return False
    return True

def mangle_name(nameobj):
    """Generate a name for 'nameobj' that is unique in the flat namespace of the
    inc module.

    """

    if isinstance(nameobj, dast.Event):
        # Event.name is already unique
        return nameobj.name

    namelist = []
    scope = nameobj.scope
    proc = scope.immediate_container_of_type(dast.Process)
    if proc is not None:
        namelist.append(proc.name)
        namelist.append('_')
    if scope is not proc and hasattr(scope, 'name'):
        namelist.append(scope.name)
        namelist.append('_')
    namelist.append(nameobj.name)
    return ''.join(namelist)

PREAMBLE = """
import da
{jbstyle_import}
{additional_directives}
ReceivedEvent = da.pat.ReceivedEvent
SentEvent = da.pat.SentEvent
{self_name} = None
{witness_var} = None
JbStyle = {is_jbstyle}
"""

# Additional directives needed by jbstyle:
JBSTYLE_DIRECTIVES = """
{0}.OPTIONS(
    deadcode_keepvars = ['JbStyle', '{1}']
)
""".format(JB_STYLE_MODULE, GLOBAL_WITNESS_VAR)

GLOBAL_READ = "globals()['{0}']"
GLOBAL_WRITE = "globals()['{0}'] = {1}"

def extract_all_queries(distalgo_ast):
    """Return a list of all queries in given ast node."""

    quex = QueryExtractor()
    quex.visit(distalgo_ast)
    return quex.queries

def extract_query_parameters(query):
    """Return set of parameters of 'query'."""

    return set(nobj for nobj in query.ordered_nameobjs
               if query.is_child_of(nobj.scope)
               # NOTE: this assumes a name can not both be a parameter
               # and assigned to inside the query, which is not true
               # for Python in general, but is true for DistAlgo
               # queries that we can handle:
               if not nobj.is_assigned_in(query))

def extract_query_events(query):
    """Return set of events used in 'query'."""

    ev = EventExtractor()
    ev.visit(query)
    return ev.events

def process_query(query, state):
    """Generates stub and hook for 'query'."""

    assert isinstance(query, dast.Expression)

    iprintd("Processing %r" % query)
    qname = QUERY_STUB_FORMAT % state.counter
    state.counter += 1
    params = list(extract_query_parameters(query))
    events = list(extract_query_events(query))
    state.parameters.update(params)
    state.events.update(events)
    iig = IncInterfaceGenerator(params)
    incqu = iig.visit(query)
    if iig.witness_set is None:
        return _process_nonwitness_query(qname, params, events, query, incqu)
    else:
        return _process_witness_query(qname, params, events,
                                      query, incqu, iig.witness_set)

def _process_nonwitness_query(stub_name, params, events, query, inc_query):
    qrydef = pyFunctionDef(
        name=stub_name,
        args=([mangle_name(nobj) for nobj in params] +
              [evt.name for evt in events]),
        decorator_list=[],
        returns=None,
        body=[Expr(Str(to_source(query.ast))),
              Return(inc_query)])
    # Replace the query node in the main module with the hook:
    qryhook = pyCall(
        func=pyAttr(INC_MODULE_VAR, stub_name),
        args=[],
        keywords=([(mangle_name(arg), PythonGenerator().visit(arg))
                   for arg in params] +
                  [(evt.name, pyAttr("self", evt.name)) for evt in events]))
    query.ast_override = qryhook
    # Replace the query node in the inc module with the hook:
    inchook = pyCall(
        func=stub_name,
        args=[],
        keywords=([(mangle_name(arg), pyName(mangle_name(arg)))
                   for arg in params] +
                  [(evt.name, pyName(evt.name)) for evt in events]))
    query.inc_query_override = inchook
    return qrydef

def _process_witness_query(stub_name, params, events,
                           query, inc_query, witness_set):
    # Query stub in inc:
    qrydef = pyFunctionDef(
        name=stub_name,
        args=([mangle_name(nobj) for nobj in params] +
              [evt.name for evt in events]),
        body=[Expr(Str(to_source(query.ast))),
              Global([GLOBAL_WITNESS_VAR]),
              Try([Assign([pyName(GLOBAL_WITNESS_VAR)],
                          pyCall("next", [pyCall("iter", [witness_set])]))],
                  [ExceptHandler(pyName("StopIteration"), None,
                                 [Assign([pyName(GLOBAL_WITNESS_VAR)],
                                         pyNone())])],
                  [], []),
              Return(inc_query)])

    # Witness propagation function in main:
    qryfunc = pyFunctionDef(
        name=query.name,
        # We know 'query' must be top-level, so no need for any arguments:
        args=[],
        body=[
            # IncInterfaceGenerator checks query has non-empty
            # ordered_local_freevars before assigning witness, so we are
            # guaranteed non-empty here:
            Nonlocal([fv.name for fv in query.ordered_local_freevars]),
            Assign([pyName(LOCAL_RESULT_VAR)],
                   pyCall(
                       func=pyAttr(INC_MODULE_VAR, stub_name),
                       args=[],
                       keywords=([(mangle_name(arg),
                                   PythonGenerator().visit(arg))
                                  for arg in params] +
                                 [(evt.name, pyAttr("self", evt.name))
                                  for evt in events]))),
            If(pyName(LOCAL_RESULT_VAR),
               [Assign([optimize_tuple(pyTuple(
                   [pyName(fv.name) for fv in query.ordered_local_freevars]))],
                       pyAttr(INC_MODULE_VAR, GLOBAL_WITNESS_VAR)),
                Return(pyTrue())],
               [Return(pyFalse())])])
    # Replace query with call to witness propagation function:
    query.ast_override = pyCall(func=query.name)
    if not hasattr(query, 'prebody'):
        query.prebody = []
    query.prebody.extend([Assign(
        targets=[pyName(nv.name) for nv in query.ordered_local_freevars],
        value=pyNone()),
                          qryfunc])

    # Replace the query node in the inc module with the hook:
    inchook = pyCall(
        func=stub_name,
        args=[],
        keywords=([(mangle_name(arg), pyName(mangle_name(arg)))
                   for arg in params] +
                  [(evt.name, pyName(evt.name)) for evt in events]))
    query.inc_query_override = inchook
    return qrydef


def process_all_queries(queries, state):
    """Generates query stubs for given queries."""

    query_stubs = [process_query(query, state) for query in queries]
    state.module.body.extend(query_stubs)

def gen_update_stub_name_for_node(updnode, state):
    """Generates an update stub name for the given update node.

    Generated name is based on the query parameters updated in 'updnode'. A
    counter is appended to the end to ensure uniqueness.

    """

    name_comps = UPDATE_STUB_PREFIX + \
                 "".join([mangle_name(nameobj)
                          for nameobj in state.updates[updnode]] +
                         [str(state.counter)])
    state.counter += 1
    return name_comps

def generate_update_stub(updnode, state):
    """Generate update stub and hook for 'updnode'."""

    uname = gen_update_stub_name_for_node(updnode, state)
    params = [nobj for nobj in updnode.nameobjs
              if updnode.is_contained_in(nobj.scope)]
    astval = IncInterfaceGenerator(params).visit(updnode)
    # the body depends on the syntactic type of update we're handling:
    if isinstance(updnode, dast.Expression):
        body = [Return(astval)]
    elif isinstance(updnode, dast.AssignmentStmt):
        body = astval
    elif isinstance(updnode, dast.DeleteStmt):
        body = astval

    updfun = FunctionDef(name=uname,
                         args=arguments(
                             args=([arg(mangle_name(nobj), None)
                                    for nobj in params]),
                             vararg=None,
                             kwonlyargs=[],<commit_msg>incgen: for jbstyle, don't return value from updates that we know won't be used.<commit_after># Copyright (c) 2010-2014 <NAME>
# Copyright (c) 2010-2014 <NAME>
# Copyright (c) 2010-2014 Stony Brook University
# Copyright (c) 2010-2014 The Research Foundation of SUNY
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import sys

from ast import *
from . import dast
from .pygen import *
from .parser import Pattern2Constant
from .utils import printe, printd, printw, OptionsManager, to_source

INC_MODULE_VAR = "IncModule"

QUERY_STUB_FORMAT = "Query_%d"
ASSIGN_STUB_FORMAT = "Assign_%s"
DEL_STUB_FORMAT = "Delete_%s"
RESET_STUB_FORMAT = "Reset_%s_Events"
INIT_STUB_PREFIX = "Init_"
UPDATE_STUB_PREFIX = "Update_"
LOCAL_WITNESS_SET = "__witness_set__"
LOCAL_RESULT_VAR = "__result__"
GLOBAL_WITNESS_VAR = "Witness"
SELF_ID_NAME = "SELF_ID"
JB_STYLE_MODULE = "invinc.runtime"
JB_STYLE_SET = JB_STYLE_MODULE + ".Set"
JB_STYLE_MAP = JB_STYLE_MODULE + ".Map"

NegatedOperatorMap = {
    dast.EqOp : NotEq,
    dast.NotEqOp : Eq,
    dast.LtOp : Gt,
    dast.LtEOp : GtE,
    dast.GtOp : Lt,
    dast.GtEOp : LtE,
    dast.IsOp : IsNot,
    dast.IsNotOp : Is,
    dast.InOp : NotIn,
    dast.NotInOp : In
}

##################################################
# Global states:

Options = None                  # Command line options
ModuleFilename = ""

##########
# Auxiliary methods:

def iprintd(message):
    printd(message, filename=ModuleFilename)

def iprintw(message):
    printw(message, filename=ModuleFilename)

def combine_not_comparison(node):
    if (isinstance(node, dast.LogicalExpr) and
            node.operator is dast.NotOp and
            isinstance(node.left, dast.ComparisonExpr)):
        exp = dast.ComparisonExpr(parent=node.parent, ast=node.ast)
        exp.left = node.left.left
        exp.right = node.left.right
        exp.left._parent = exp
        exp.right._parent = exp
        exp.comparator = NegatedOperatorMap[node.left.comparator]
        return exp
    else:
        return node

def apply_demorgan_rule(node):
    if (isinstance(node, dast.LogicalExpr) and
            node.operator is dast.NotOp and
            isinstance(node.left, dast.LogicalExpr) and
            node.left.operator in {dast.AndOp, dast.OrOp}):
        exp = dast.LogicalExpr(parent=node.parent, ast=node.ast,
                               op=NegatedOperatorMap[node.left.operator])
        exp.subexprs = [dast.LogicalExpr(parent=exp, ast=None,
                                         op=dast.NotOp,
                                         subexprs=[e])
                        for e in node.left.subexprs]
        for e in exp.subexprs:
            e.left._parent = e
        return exp
    else:
        return node

def domain_for_condition(domainspec, condition):
    expr = dast.SetCompExpr(domainspec.parent)
    expr.conditions.append(domainspec.clone())
    expr.conditions.append(condition)
    if len(domainspec.pattern.ordered_freevars) == 1:
        expr.elem = dast.SimpleExpr(expr)
        expr.elem.value = domainspec.pattern.ordered_freevars[0]
        domainspec.pattern = dast.PatternExpr(domainspec)
        domainspec.pattern.pattern = dast.FreePattern(
            domainspec.pattern, value=expr.elem.value)
    else:
        expr.elem = dast.TupleExpr(expr)
        expr.elem.subexprs = [dast.SimpleExpr(expr.elem, value=v)
                              for v in domainspec.pattern.ordered_freevars]
        domainspec.pattern = dast.PatternExpr(domainspec)
        domainspec.pattern.pattern = dast.TuplePattern(
            domainspec.pattern,
            value = [dast.FreePattern(domainspec.pattern, value=fv.value)
                     for fv in expr.elem.subexprs])
    domainspec.domain = expr
    iprintd("domain_for_condition: " + str(expr))
    return expr

def optimize_tuple(elt):
    """Expand single element tuples."""

    if type(elt) is Tuple and len(elt.elts) == 1:
        elt = elt.elts[0]
    return elt

def ast_eq(left, right):
    """Structural equality of AST nodes."""

    if type(left) is not type(right):
        return False
    if isinstance(left, list):
        if len(left) != len(right):
            return False
        for litem, ritem in zip(left, right):
            if not ast_eq(litem, ritem):
                return False
        return True
    if not isinstance(left, AST):
        return left == right
    for (left_fieldname, left_value), (right_fieldname, right_value) in \
            zip(iter_fields(left), iter_fields(right)):
        if not (left_fieldname == right_fieldname):
            break
        if not ast_eq(left_value, right_value):
            break
    else:
        return True
    return False

def is_all_wildcards(targets):
    """True if 'targets' contain only wildcards."""

    for elt in targets:
        if not (isinstance(elt, Name) and elt.id == '_'):
            return False
    return True

def mangle_name(nameobj):
    """Generate a name for 'nameobj' that is unique in the flat namespace of the
    inc module.

    """

    if isinstance(nameobj, dast.Event):
        # Event.name is already unique
        return nameobj.name

    namelist = []
    scope = nameobj.scope
    proc = scope.immediate_container_of_type(dast.Process)
    if proc is not None:
        namelist.append(proc.name)
        namelist.append('_')
    if scope is not proc and hasattr(scope, 'name'):
        namelist.append(scope.name)
        namelist.append('_')
    namelist.append(nameobj.name)
    return ''.join(namelist)

PREAMBLE = """
import da
{jbstyle_import}
{additional_directives}
ReceivedEvent = da.pat.ReceivedEvent
SentEvent = da.pat.SentEvent
{self_name} = None
{witness_var} = None
JbStyle = {is_jbstyle}
"""

# Additional directives needed by jbstyle:
JBSTYLE_DIRECTIVES = """
{0}.OPTIONS(
    deadcode_keepvars = ['JbStyle', '{1}']
)
""".format(JB_STYLE_MODULE, GLOBAL_WITNESS_VAR)

GLOBAL_READ = "globals()['{0}']"
GLOBAL_WRITE = "globals()['{0}'] = {1}"

def extract_all_queries(distalgo_ast):
    """Return a list of all queries in given ast node."""

    quex = QueryExtractor()
    quex.visit(distalgo_ast)
    return quex.queries

def extract_query_parameters(query):
    """Return set of parameters of 'query'."""

    return set(nobj for nobj in query.ordered_nameobjs
               if query.is_child_of(nobj.scope)
               # NOTE: this assumes a name can not both be a parameter
               # and assigned to inside the query, which is not true
               # for Python in general, but is true for DistAlgo
               # queries that we can handle:
               if not nobj.is_assigned_in(query))

def extract_query_events(query):
    """Return set of events used in 'query'."""

    ev = EventExtractor()
    ev.visit(query)
    return ev.events

def process_query(query, state):
    """Generates stub and hook for 'query'."""

    assert isinstance(query, dast.Expression)

    iprintd("Processing %r" % query)
    qname = QUERY_STUB_FORMAT % state.counter
    state.counter += 1
    params = list(extract_query_parameters(query))
    events = list(extract_query_events(query))
    state.parameters.update(params)
    state.events.update(events)
    iig = IncInterfaceGenerator(params)
    incqu = iig.visit(query)
    if iig.witness_set is None:
        return _process_nonwitness_query(qname, params, events, query, incqu)
    else:
        return _process_witness_query(qname, params, events,
                                      query, incqu, iig.witness_set)

def _process_nonwitness_query(stub_name, params, events, query, inc_query):
    qrydef = pyFunctionDef(
        name=stub_name,
        args=([mangle_name(nobj) for nobj in params] +
              [evt.name for evt in events]),
        decorator_list=[],
        returns=None,
        body=[Expr(Str(to_source(query.ast))),
              Return(inc_query)])
    # Replace the query node in the main module with the hook:
    qryhook = pyCall(
        func=pyAttr(INC_MODULE_VAR, stub_name),
        args=[],
        keywords=([(mangle_name(arg), PythonGenerator().visit(arg))
                   for arg in params] +
                  [(evt.name, pyAttr("self", evt.name)) for evt in events]))
    query.ast_override = qryhook
    # Replace the query node in the inc module with the hook:
    inchook = pyCall(
        func=stub_name,
        args=[],
        keywords=([(mangle_name(arg), pyName(mangle_name(arg)))
                   for arg in params] +
                  [(evt.name, pyName(evt.name)) for evt in events]))
    query.inc_query_override = inchook
    return qrydef

def _process_witness_query(stub_name, params, events,
                           query, inc_query, witness_set):
    # Query stub in inc:
    qrydef = pyFunctionDef(
        name=stub_name,
        args=([mangle_name(nobj) for nobj in params] +
              [evt.name for evt in events]),
        body=[Expr(Str(to_source(query.ast))),
              Global([GLOBAL_WITNESS_VAR]),
              Try([Assign([pyName(GLOBAL_WITNESS_VAR)],
                          pyCall("next", [pyCall("iter", [witness_set])]))],
                  [ExceptHandler(pyName("StopIteration"), None,
                                 [Assign([pyName(GLOBAL_WITNESS_VAR)],
                                         pyNone())])],
                  [], []),
              Return(inc_query)])

    # Witness propagation function in main:
    qryfunc = pyFunctionDef(
        name=query.name,
        # We know 'query' must be top-level, so no need for any arguments:
        args=[],
        body=[
            # IncInterfaceGenerator checks query has non-empty
            # ordered_local_freevars before assigning witness, so we are
            # guaranteed non-empty here:
            Nonlocal([fv.name for fv in query.ordered_local_freevars]),
            Assign([pyName(LOCAL_RESULT_VAR)],
                   pyCall(
                       func=pyAttr(INC_MODULE_VAR, stub_name),
                       args=[],
                       keywords=([(mangle_name(arg),
                                   PythonGenerator().visit(arg))
                                  for arg in params] +
                                 [(evt.name, pyAttr("self", evt.name))
                                  for evt in events]))),
            If(pyName(LOCAL_RESULT_VAR),
               [Assign([optimize_tuple(pyTuple(
                   [pyName(fv.name) for fv in query.ordered_local_freevars]))],
                       pyAttr(INC_MODULE_VAR, GLOBAL_WITNESS_VAR)),
                Return(pyTrue())],
               [Return(pyFalse())])])
    # Replace query with call to witness propagation function:
    query.ast_override = pyCall(func=query.name)
    if not hasattr(query, 'prebody'):
        query.prebody = []
    query.prebody.extend([Assign(
        targets=[pyName(nv.name) for nv in query.ordered_local_freevars],
        value=pyNone()),
                          qryfunc])

    # Replace the query node in the inc module with the hook:
    inchook = pyCall(
        func=stub_name,
        args=[],
        keywords=([(mangle_name(arg), pyName(mangle_name(arg)))
                   for arg in params] +
                  [(evt.name, pyName(evt.name)) for evt in events]))
    query.inc_query_override = inchook
    return qrydef


def process_all_queries(queries, state):
    """Generates query stubs for given queries."""

    query_stubs = [process_query(query, state) for query in queries]
    state.module.body.extend(query_stubs)

def gen_update_stub_name_for_node(updnode, state):
    """Generates an update stub name for the given update node.

    Generated name is based on the query parameters updated in 'updnode'. A
    counter is appended to the end to ensure uniqueness.

    """

    name_comps = UPDATE_STUB_PREFIX + \
                 "".join([mangle_name(nameobj)
                          for nameobj in state.updates[updnode]] +
                         [str(state.counter)])
    state.counter += 1
    return name_comps

def generate_update_stub(updnode, state):
    """Generate update stub and hook for 'updnode'."""

    uname = gen_update_stub_name_for_node(updnode, state)
    params = [nobj for nobj in updnode.nameobjs
              if updnode.is_contained_in(nobj.scope)]
    astval = IncInterfaceGenerator(params).visit(updnode)
    # the body depends on the syntactic type of update we're handling:
    if isinstance(updnode, dast.Expression):
        if Options.jb_style and type(updnode.parent) is dast.SimpleStmt:
            # jbstyle doesn't like updates as expressions, so just don't
            # return anything and hope for the best:
            body = [Expr(astval)]
        else:
            body = [Return(astval)]
    elif isinstance(updnode, dast.AssignmentStmt):
        body = astval
    elif isinstance(updnode, dast.DeleteStmt):
        body = astval

    updfun = FunctionDef(name=uname,
                         args=arguments(
                             args=([arg(mangle_name(nobj), None)
                                    for nobj in params]),
                             vararg=None,
                             kwonlyargs=[],
{'fraction_non_alphanumeric': 0.10240748678382626, 'fraction_numerical': 0.0031433061865980853, 'mean_word_length': 2.4369015467714217, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1962377', 'n_tokens_mistral': 8538, 'n_tokens_neox': 8251, 'n_words': 2211}