<commit_before>import io
import json
import pandas
import os
import re
import struct
import sys


def wrap_value(val):
    if val is None:
        return {"type": "null"}
    elif isinstance(val, Exception):
        return {"type": "exception",
                "class": type(val).__name__,
                "args": val.args}
    elif isinstance(val, pandas.DataFrame):
        return {"type": "dataframe",
                "csv": val.to_csv(index=False)}
    else:
        return {"type": "primitive",
                "value": val}


def wrap_bindings(bindings):
    return [{"name": n, **wrap_value(v)}
            for n, v in bindings.items()]


class Rattlr:
    assignment = re.compile("^\\s*([_a-zA-Z]\\w*)\\s*=\\s*(.+)$")
    undefined = re.compile("^name '(.+)' is not defined$")
    import_simple = re.compile("^\\s*import\\s+(\\S+)$")
    import_as = re.compile("^\\s*import\\s+(\\S+)\\s+as\\s+([_a-zA-Z]\\w*)$")

    def __init__(self, in_pipe, out_pipe):
        self.in_pipe = in_pipe
        self.out_pipe = out_pipe
        self.persistent = {}

    def receive(self):
        size = struct.unpack('i', self.in_pipe.read(4))[0]
        if size == 0:
            return None
        json_data = self.in_pipe.read(size)[:-1]
        data = json.loads(json_data)
        if "value" in data:
            if "dataframe" in data["type"]:
                str_in = io.StringIO(data["value"][0])
                data["value"] = pandas.read_csv(str_in)
        return data

    def send(self, reply):
        try:
            reply_data = bytes(json.dumps(reply), "utf-8")
            self.out_pipe.write(struct.pack("i", len(reply_data)))
            self.out_pipe.write(reply_data)
            self.out_pipe.flush()
        except Exception as exc:
            self.send(wrap_value(exc))

    def request(self, name):
        req = {"type": "request",
               "name": name}
        self.send(req)
        return self.receive()

    def eval_expr(self, e, imports, bindings, from_r):
        m = Rattlr.assignment.match(e)
        while True:
            lcls = {**self.persistent, **bindings, **from_r}
            try:
                if m:
                    name = m.group(1)
                    res = eval(m.group(2), lcls, imports)
                    if name[0] == '_':
                        self.persistent[name] = res
                    else:
                        bindings[name] = res
                    return None
                else:
                    return eval(e, lcls, imports)

            except NameError as err:
                n = Rattlr.undefined.match(err.args[0])
                if n:
                    name = n.group(1)
                    req = self.request(name)
                    if "missing" in req:
                        raise err
                    else:
                        from_r[name] = req["value"]
                else:
                    raise err

    def eval_sequence(self, data):
        imports = {i: __import__(i) for i in data['imports']}

        try:
            bindings = {}
            from_r = {}
            for e in data['exprs']:
                m = Rattlr.import_simple.match(e)
                if m:
                    self.persistent[m.group(1)] = __import__(m.group(1))
                    val = None
                    continue

                m = Rattlr.import_as.match(e)
                if m:
                    self.persistent[m.group(2)] = __import__(m.group(1))
                    val = None
                    continue

                val = self.eval_expr(e, imports, bindings, from_r)
        except Exception as exc:
            val = exc

        return {**wrap_value(val), "bindings": wrap_bindings(bindings)}

    def receive_eval_send(self):
        data = self.receive()
        if data is None:
            return False

        reply = self.eval_sequence(data)

        self.send(reply)
        return True


def main(pipe_dir):
    with open(os.path.join(pipe_dir, "rToPython"), "rb") as in_pipe:
        with open(os.path.join(pipe_dir, "pythonToR"), "wb") as out_pipe:
            print("python started", file=sys.stderr)
            rattlr = Rattlr(in_pipe, out_pipe)
            while rattlr.receive_eval_send():
                pass
    print("python shutdown", file=sys.stderr)


if __name__ == '__main__':
    main(pipe_dir=sys.argv[1])<commit_msg>move expression evaluation into classes<commit_after>import io
import json
import numpy
import os
import pandas
import re
import struct
import sys


def wrap_value(val):
    if val is None:
        return {"type": "null"}
    elif isinstance(val, Exception):
        return {"type": "exception",
                "class": type(val).__name__,
                "args": val.args}
    elif isinstance(val, numpy.int64):
        return {"type": "primitive",
                "value": numpy.asscalar(val)}
    elif isinstance(val, numpy.ndarray):
        return {"type": "primitive",
                "value": val.tolist()}
    elif isinstance(val, pandas.DataFrame):
        return {"type": "dataframe",
                "csv": val.to_csv(index=False)}
    else:
        return {"type": "primitive",
                "value": val}


def wrap_bindings(bindings):
    return [{"name": n, **wrap_value(v)}
            for n, v in bindings.items()]


class Environment:
    def __init__(self, rattlr, imports, bindings, from_r):
        self.rattlr = rattlr
        self.bindings = bindings
        self.from_r = from_r
        self.imports = imports

    def make_locals(self):
        return {**self.from_r, **self.rattlr.persistent, **self.bindings}

    def evaluate(self, expr):
        while True:
            try:
                return eval(expr, self.make_locals(), self.imports)
            except NameError as err:
                n = Rattlr.undefined.match(err.args[0])
                if n:
                    name = n.group(1)
                    req = self.rattlr.request(name)
                    if "missing" in req:
                        raise err
                    else:
                        self.from_r[name] = req["value"]
                else:
                    raise err


class Expression:
    def __init__(self, envir):
        self.envir = envir

    def evaluate(self):
        pass


class SimpleExpr(Expression):
    def __init__(self, expr, envir):
        Expression.__init__(self, envir)
        self.expr = expr

    def evaluate(self):
        return self.envir.evaluate(self.expr)


class Assignment(Expression):
    def __init__(self, name, expr, envir):
        Expression.__init__(self, envir)
        self.name = name
        self.expr = SimpleExpr(expr, envir)

    def evaluate(self):
        res = self.expr.evaluate()
        if self.name[0] == '_':
            self.persistent[self.name] = res
        else:
            self.envir.bindings[self.name] = res
        return None


class SimpleImport(Expression):
    def __init__(self, package, envir):
        Expression.__init__(self, envir)
        self.package = package

    def evaluate(self):
        self.envir.rattlr.persistent[self.package] = __import__(self.package)


class ImportAs(Expression):
    def __init__(self, name, package, envir):
        Expression.__init__(self, envir)
        self.name = name
        self.package = package

    def evaluate(self):
        self.envir.rattlr.persistent[self.name] = __import__(self.package)


class Rattlr:
    assignment = re.compile("^\\s*([_a-zA-Z]\\w*)\\s*=\\s*(.+)$")
    undefined = re.compile("^name '(.+)' is not defined$")
    import_simple = re.compile("^\\s*import\\s+(\\S+)$")
    import_as = re.compile("^\\s*import\\s+(\\S+)\\s+as\\s+([_a-zA-Z]\\w*)$")

    def __init__(self, in_pipe, out_pipe):
        self.in_pipe = in_pipe
        self.out_pipe = out_pipe
        self.persistent = {}

    def receive(self):
        size = struct.unpack('i', self.in_pipe.read(4))[0]
        if size == 0:
            return None
        json_data = self.in_pipe.read(size)[:-1]
        data = json.loads(json_data)
        if "value" in data:
            if "dataframe" in data["type"]:
                str_in = io.StringIO(data["value"][0])
                data["value"] = pandas.read_csv(str_in)
            elif isinstance(data["value"], list):
                data["value"] = numpy.asarray(data["value"])

        return data

    def send(self, reply):
        try:
            reply_data = bytes(json.dumps(reply), "utf-8")
            self.out_pipe.write(struct.pack("i", len(reply_data)))
            self.out_pipe.write(reply_data)
            self.out_pipe.flush()
        except Exception as exc:
            self.send(wrap_value(exc))

    def request(self, name):
        req = {"type": "request",
               "name": name}
        self.send(req)
        return self.receive()

    def make_expr(self, e, imports, bindings, from_r):
        envir = Environment(self, imports, bindings, from_r)
        m = Rattlr.assignment.match(e)
        if m:
            return Assignment(m.group(1), m.group(2), envir)
        m = Rattlr.import_simple.match(e)
        if m:
            return SimpleImport(m.group(1), envir)
        m = Rattlr.import_as.match(e)
        if m:
            return ImportAs(m.group(2), m.group(1), envir)
        return SimpleExpr(e, envir)

    def eval_sequence(self, data):
        imports = {i: __import__(i) for i in data['imports']}
        try:
            bindings = {}
            from_r = {}
            val = None
            for e in data['exprs']:
                expr = self.make_expr(e, imports, bindings, from_r)
                val = expr.evaluate()
        except Exception as exc:
            val = exc

        return {**wrap_value(val), "bindings": wrap_bindings(bindings)}

    def receive_eval_send(self):
        data = self.receive()
        if data is None:
            return False

        reply = self.eval_sequence(data)

        self.send(reply)
        return True


def main(pipe_dir):
    with open(os.path.join(pipe_dir, "rToPython"), "rb") as in_pipe:
        with open(os.path.join(pipe_dir, "pythonToR"), "wb") as out_pipe:
            print("python started", file=sys.stderr)
            rattlr = Rattlr(in_pipe, out_pipe)
            while rattlr.receive_eval_send():
                pass
    print("python shutdown", file=sys.stderr)


if __name__ == '__main__':
    main(pipe_dir=sys.argv[1])
{'fraction_non_alphanumeric': 0.13085730780332916, 'fraction_numerical': 0.0031752140864043108, 'mean_word_length': 1.777659005879209, 'pattern_counts': {'":': 28, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5247619', 'n_tokens_mistral': 3279, 'n_tokens_neox': 3127, 'n_words': 708}