<commit_before>from . import codegen
from .. import const
from ..parser import tree
from ..parser import syntax

# Choose a function based on the number of arguments.
varary  = lambda *fs: lambda *xs: fs[len(xs) - 1](*xs)


class Compiler:

    def __init__(self):

        super().__init__()

        # A mutable code object currently in use.
        self.code = None

        # An expression currently being processed by innermost `load`.
        self._loading = None

        # Predefined stuff that translates to bytecode.
        self.builtins = {
            ',':  self.tuple
          , '':   self.call
          , '$':  self.call
          , ':':  self.call
          , '=':  self.store
          , '->': self.function

          , 'inherit': self.class_
          , 'return':  lambda a: (self.opcode('DUP_TOP', a), self.code.RETURN_VALUE())
          , 'yield':   lambda a: self.opcode('YIELD_VALUE',  a)

          , 'not': lambda a: self.opcode('UNARY_NOT',    a)
          , '~':   lambda a: self.opcode('UNARY_INVERT', a)

          , '+': varary(
                lambda a:    self.opcode('UNARY_POSITIVE', a)
              , lambda a, b: self.opcode('BINARY_ADD',     a, b)
            )

          , '-': varary(
                lambda a:    self.opcode('UNARY_NEGATIVE',  a)
              , lambda a, b: self.opcode('BINARY_SUBTRACT', a, b)
            )

          , '<':   lambda a, b: self.opcode('COMPARE_OP', a, b, arg='<')
          , '<=':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='<=')
          , '==':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='==')
          , '!=':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='!=')
          , '>':   lambda a, b: self.opcode('COMPARE_OP', a, b, arg='>')
          , '>=':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='>=')
          , 'is':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='is')
          , 'in':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='in')

          , 'or':  lambda a, b: (self.load(a), self.code.JUMP_IF_TRUE_OR_POP (delta=-1), self.load(b))[1]()
          , 'and': lambda a, b: (self.load(a), self.code.JUMP_IF_FALSE_OR_POP(delta=-1), self.load(b))[1]()

          , '.':   lambda a, b: self.opcode('LOAD_ATTR',            a, arg=b)
          , '!!':  lambda a, b: self.opcode('BINARY_SUBSCR',        a, b)
          , '*':   lambda a, b: self.opcode('BINARY_MULTIPLY',      a, b)
          , '**':  lambda a, b: self.opcode('BINARY_POWER',         a, b)
          , '/':   lambda a, b: self.opcode('BINARY_TRUE_DIVIDE',   a, b)
          , '//':  lambda a, b: self.opcode('BINARY_FLOOR_DIVIDE',  a, b)
          , '%':   lambda a, b: self.opcode('BINARY_MODULO',        a, b)
          , '&':   lambda a, b: self.opcode('BINARY_AND',           a, b)
          , '^':   lambda a, b: self.opcode('BINARY_XOR',           a, b)
          , '|':   lambda a, b: self.opcode('BINARY_OR',            a, b)
          , '<<':  lambda a, b: self.opcode('BINARY_LSHIFT',        a, b)
          , '>>':  lambda a, b: self.opcode('BINARY_RSHIFT',        a, b)
          , '!!=': lambda a, b: self.opcode('BINARY_SUBSCR',        a, b, inplace=True)
          , '+=':  lambda a, b: self.opcode('INPLACE_ADD',          a, b, inplace=True)
          , '-=':  lambda a, b: self.opcode('INPLACE_SUBTRACT',     a, b, inplace=True)
          , '*=':  lambda a, b: self.opcode('INPLACE_MULTIPLY',     a, b, inplace=True)
          , '**=': lambda a, b: self.opcode('INPLACE_POWER',        a, b, inplace=True)
          , '/=':  lambda a, b: self.opcode('INPLACE_TRUE_DIVIDE',  a, b, inplace=True)
          , '//=': lambda a, b: self.opcode('INPLACE_FLOOR_DIVIDE', a, b, inplace=True)
          , '%=':  lambda a, b: self.opcode('INPLACE_MODULO',       a, b, inplace=True)
          , '&=':  lambda a, b: self.opcode('INPLACE_AND',          a, b, inplace=True)
          , '^=':  lambda a, b: self.opcode('INPLACE_XOR',          a, b, inplace=True)
          , '|=':  lambda a, b: self.opcode('INPLACE_OR',           a, b, inplace=True)
          , '<<=': lambda a, b: self.opcode('INPLACE_LSHIFT',       a, b, inplace=True)
          , '>>=': lambda a, b: self.opcode('INPLACE_RSHIFT',       a, b, inplace=True)
          , '.~':  lambda a, b: self.opcode('DELETE_ATTR',          a, arg=b, delta=0, ret=None)
          , '!!~': lambda a, b: self.opcode('DELETE_SUBSCR',        a, b,     delta=0, ret=None)
        }

    def opcode(self, opcode, *args, arg=0, delta=1, inplace=False, ret=None):

        self.load(*args)
        self.code.append(opcode, arg, -len(args) + delta)
        inplace and self.store_top(*syntax.assignment_target(args[0]))
        inplace and self.load(ret)

    def tuple(self, lhs, *rhs):

        args = syntax.tuple(lhs, *rhs)
        self.opcode('BUILD_TUPLE', *args, arg=len(args))

    def function(self, args, code, hook=0):

        args, kwargs, defs, kwdefs, varargs, varkwargs, code = syntax.function(args, code)
        self.load_map(kwdefs)<commit_msg>Remove `ret` argument of `Compiler.opcode`.<commit_after>from . import codegen
from .. import const
from ..parser import tree
from ..parser import syntax

# Choose a function based on the number of arguments.
varary  = lambda *fs: lambda *xs: fs[len(xs) - 1](*xs)


class Compiler:

    def __init__(self):

        super().__init__()

        # A mutable code object currently in use.
        self.code = None

        # An expression currently being processed by innermost `load`.
        self._loading = None

        # Predefined stuff that translates to bytecode.
        self.builtins = {
            ',':  self.tuple
          , '':   self.call
          , '$':  self.call
          , ':':  self.call
          , '=':  self.store
          , '->': self.function

          , 'inherit': self.class_
          , 'return':  lambda a: (self.opcode('DUP_TOP', a), self.code.RETURN_VALUE())
          , 'yield':   lambda a: self.opcode('YIELD_VALUE',  a)

          , 'not': lambda a: self.opcode('UNARY_NOT',    a)
          , '~':   lambda a: self.opcode('UNARY_INVERT', a)

          , '+': varary(
                lambda a:    self.opcode('UNARY_POSITIVE', a)
              , lambda a, b: self.opcode('BINARY_ADD',     a, b)
            )

          , '-': varary(
                lambda a:    self.opcode('UNARY_NEGATIVE',  a)
              , lambda a, b: self.opcode('BINARY_SUBTRACT', a, b)
            )

          , '<':   lambda a, b: self.opcode('COMPARE_OP', a, b, arg='<')
          , '<=':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='<=')
          , '==':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='==')
          , '!=':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='!=')
          , '>':   lambda a, b: self.opcode('COMPARE_OP', a, b, arg='>')
          , '>=':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='>=')
          , 'is':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='is')
          , 'in':  lambda a, b: self.opcode('COMPARE_OP', a, b, arg='in')

          , 'or':  lambda a, b: (self.load(a), self.code.JUMP_IF_TRUE_OR_POP (delta=-1), self.load(b))[1]()
          , 'and': lambda a, b: (self.load(a), self.code.JUMP_IF_FALSE_OR_POP(delta=-1), self.load(b))[1]()

          , '.':   lambda a, b: self.opcode('LOAD_ATTR',            a, arg=b)
          , '!!':  lambda a, b: self.opcode('BINARY_SUBSCR',        a, b)
          , '*':   lambda a, b: self.opcode('BINARY_MULTIPLY',      a, b)
          , '**':  lambda a, b: self.opcode('BINARY_POWER',         a, b)
          , '/':   lambda a, b: self.opcode('BINARY_TRUE_DIVIDE',   a, b)
          , '//':  lambda a, b: self.opcode('BINARY_FLOOR_DIVIDE',  a, b)
          , '%':   lambda a, b: self.opcode('BINARY_MODULO',        a, b)
          , '&':   lambda a, b: self.opcode('BINARY_AND',           a, b)
          , '^':   lambda a, b: self.opcode('BINARY_XOR',           a, b)
          , '|':   lambda a, b: self.opcode('BINARY_OR',            a, b)
          , '<<':  lambda a, b: self.opcode('BINARY_LSHIFT',        a, b)
          , '>>':  lambda a, b: self.opcode('BINARY_RSHIFT',        a, b)
          , '!!=': lambda a, b: self.opcode('BINARY_SUBSCR',        a, b, inplace=True)
          , '+=':  lambda a, b: self.opcode('INPLACE_ADD',          a, b, inplace=True)
          , '-=':  lambda a, b: self.opcode('INPLACE_SUBTRACT',     a, b, inplace=True)
          , '*=':  lambda a, b: self.opcode('INPLACE_MULTIPLY',     a, b, inplace=True)
          , '**=': lambda a, b: self.opcode('INPLACE_POWER',        a, b, inplace=True)
          , '/=':  lambda a, b: self.opcode('INPLACE_TRUE_DIVIDE',  a, b, inplace=True)
          , '//=': lambda a, b: self.opcode('INPLACE_FLOOR_DIVIDE', a, b, inplace=True)
          , '%=':  lambda a, b: self.opcode('INPLACE_MODULO',       a, b, inplace=True)
          , '&=':  lambda a, b: self.opcode('INPLACE_AND',          a, b, inplace=True)
          , '^=':  lambda a, b: self.opcode('INPLACE_XOR',          a, b, inplace=True)
          , '|=':  lambda a, b: self.opcode('INPLACE_OR',           a, b, inplace=True)
          , '<<=': lambda a, b: self.opcode('INPLACE_LSHIFT',       a, b, inplace=True)
          , '>>=': lambda a, b: self.opcode('INPLACE_RSHIFT',       a, b, inplace=True)
          , '.~':  lambda a, b: (self.opcode('DELETE_ATTR',         a, arg=b, delta=0), self.load(None))
          , '!!~': lambda a, b: (self.opcode('DELETE_SUBSCR',       a, b,     delta=0), self.load(None))
        }

    def opcode(self, opcode, *args, arg=0, delta=1, inplace=False, ret=None):

        self.load(*args)
        self.code.append(opcode, arg, -len(args) + delta)
        inplace and self.store_top(*syntax.assignment_target(args[0]))

    def tuple(self, lhs, *rhs):

        args = syntax.tuple(lhs, *rhs)
        self.opcode('BUILD_TUPLE', *args, arg=len(args))

    def function(self, args, code, hook=0):

        args, kwargs, defs, kwdefs, varargs, varkwargs, code = syntax.function(args, code)
        self.load_map(kwdefs)
{'fraction_non_alphanumeric': 0.1854072963518241, 'fraction_numerical': 0.0021989005497251375, 'mean_word_length': 2.0968740328071807, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3574719', 'n_tokens_mistral': 3636, 'n_tokens_neox': 3655, 'n_words': 814}