<commit_before>#include <Python.h>
#include <opcode.h>
#include <marshal.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>

#include "reval.h"
#include "rcompile.h"

#ifdef FALCON_DEBUG
static bool logging_enabled() {
  static bool _is_logging = getenv("EVAL_LOG") != NULL;
  return _is_logging;
}

#define EVAL_LOG(...) if (logging_enabled()) { fprintf(stderr, __VA_ARGS__); fputs("\n", stderr); }
#define CHECK_VALID(obj) { Reg_AssertGt(obj->ob_refcnt, 0); }
#else
#define EVAL_LOG(...)
#define CHECK_VALID(obj) { }
//#define CHECK_VALID(obj) Reg_AssertGt(obj->ob_refcnt, 0);
#endif

#define REUSE_INT_REGISTERS 0

typedef long (*IntegerBinaryOp)(long, long);
typedef PyObject* (*PythonBinaryOp)(PyObject*, PyObject*);
typedef PyObject* (*UnaryFunction)(PyObject*);

struct GilHelper {
  PyGILState_STATE state_;

  GilHelper() :
      state_(PyGILState_Ensure()) {
  }
  ~GilHelper() {
    PyGILState_Release(state_);
  }
};

struct RefHelper {
  PyObject* obj;

  f_inline RefHelper(PyObject* o) :
      obj(o) {
    Py_INCREF(obj);
  }
  f_inline ~RefHelper() {
    Py_DECREF(obj);
  }
};

// Catch bad case of RefHelper(x) (goes immediately out of scope
#define GilHelper(v) static int MustInitWithVar[-1];
#define RefHelper(v) static int MustInitWithVar[-1];

// Wrapper around PyXXXObject* which allows implicit casting to PyObject.
// This let's us access member variables and call API functions easily.
template<class T>
struct PyObjHelper {
  const T& val_;
  PyObjHelper(const T& t) :
      val_(t) {
  }
  operator PyObject*() {
    return (PyObject*) val_;
  }
  operator T() {
    return val_;
  }
  operator void*() {
    return val_;
  }

  bool operator==(void* v) const {
    return val_ == v;
  }

  bool operator!=(void* v) const {
    return val_ != v;
  }

  T operator->() {
    return val_;
  }
};

RegisterFrame::RegisterFrame(RegisterCode* rcode, PyObject* obj, const ObjVector& args, const ObjVector& kw) :
    code(rcode) {
  instructions_ = code->instructions.data();
  current_hint = -1;

  if (rcode->function) {
    globals_ = PyFunction_GetGlobals(rcode->function);
    locals_ = NULL;
  } else {
    globals_ = PyEval_GetGlobals();
    locals_ = PyEval_GetGlobals();
  }

  Reg_Assert(kw.empty(), "Keyword args not supported.");

  builtins_ = PyEval_GetBuiltins();

  names_ = code->names();
  consts_ = code->consts();

#if ! STACK_ALLOC_REGISTERS
  registers = new PyObject*[rcode->num_registers];
#endif

  const int num_args = args.size();
  if (rcode->num_cells > 0) {
#if ! STACK_ALLOC_REGISTERS
    freevars = new PyObject*[rcode->num_cells];
#endif
    int i;
    for (i = 0; i < rcode->num_cellvars; ++i) {
      bool found_argname = false;
      char *cellname = PyString_AS_STRING(PyTuple_GET_ITEM(rcode->code()->co_cellvars, i));
      for (int arg_idx = 0; arg_idx < num_args; ++arg_idx) {
        char* argname = PyString_AS_STRING(PyTuple_GET_ITEM(rcode->code()->co_varnames, arg_idx));
        if (strcmp(cellname, argname) == 0) {
          PyObject* arg_value = args[arg_idx];
          freevars[i] = PyCell_New(arg_value);
          found_argname = true;
          break;
        }
      }
      if (!found_argname) {
        freevars[i] = PyCell_New(NULL);
      }
    }

    PyObject* closure = ((PyFunctionObject*) rcode->function)->func_closure;
    if (closure) {
      for (int i = rcode->num_cellvars; i < rcode->num_cells; ++i) {
        freevars[i] = PyTuple_GET_ITEM(closure, i - rcode->num_cellvars) ;
        Py_INCREF(freevars[i]);
      }
    } else {
      for (int i = rcode->num_cellvars; i < rcode->num_cells; ++i) {
        freevars[i] = PyCell_New(NULL);
      }
    }
  } else {
#if ! STACK_ALLOC_REGISTERS
    freevars = NULL;
#endif
  }

  Log_Info("Alignments: reg: %d code: %d consts: %d globals: %d, this: %d",
           ((long)registers) % 64, ((long)rcode->instructions.data()) % 64, (long)consts_ % 64, (long)globals_ % 64, (long)this % 64);

  const int num_registers = code->num_registers;

  // setup const and local register aliases.
  int num_consts = PyTuple_GET_SIZE(consts());
  for (int i = 0; i < num_consts; ++i) {
    registers[i] = PyTuple_GET_ITEM(consts(), i) ;
  }

  int needed_args = code->code()->co_argcount;
  int offset = num_consts;
  if (PyMethod_Check(obj)) {
    PyObject* self = PyMethod_GET_SELF(obj);

    Reg_Assert(self != NULL, "Method call without a bound self.");
    registers[offset++] = self;
    Py_INCREF(self);

    needed_args--;
  }

  if (code->function) {
    PyObject* def_args = PyFunction_GET_DEFAULTS(code->function);
    int num_def_args = def_args == NULL ? 0 : PyTuple_GET_SIZE(def_args);<commit_msg>Reduce logging.<commit_after>#include <Python.h>
#include <opcode.h>
#include <marshal.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>

#include "reval.h"
#include "rcompile.h"

#ifdef FALCON_DEBUG
static bool logging_enabled() {
  static bool _is_logging = getenv("EVAL_LOG") != NULL;
  return _is_logging;
}

#define EVAL_LOG(...) if (logging_enabled()) { fprintf(stderr, __VA_ARGS__); fputs("\n", stderr); }
#define CHECK_VALID(obj) { Reg_AssertGt(obj->ob_refcnt, 0); }
#else
#define EVAL_LOG(...)
#define CHECK_VALID(obj) { }
//#define CHECK_VALID(obj) Reg_AssertGt(obj->ob_refcnt, 0);
#endif

#define REUSE_INT_REGISTERS 0

typedef long (*IntegerBinaryOp)(long, long);
typedef PyObject* (*PythonBinaryOp)(PyObject*, PyObject*);
typedef PyObject* (*UnaryFunction)(PyObject*);

struct GilHelper {
  PyGILState_STATE state_;

  GilHelper() :
      state_(PyGILState_Ensure()) {
  }
  ~GilHelper() {
    PyGILState_Release(state_);
  }
};

struct RefHelper {
  PyObject* obj;

  f_inline RefHelper(PyObject* o) :
      obj(o) {
    Py_INCREF(obj);
  }
  f_inline ~RefHelper() {
    Py_DECREF(obj);
  }
};

// Catch bad case of RefHelper(x) (goes immediately out of scope
#define GilHelper(v) static int MustInitWithVar[-1];
#define RefHelper(v) static int MustInitWithVar[-1];

// Wrapper around PyXXXObject* which allows implicit casting to PyObject.
// This let's us access member variables and call API functions easily.
template<class T>
struct PyObjHelper {
  const T& val_;
  PyObjHelper(const T& t) :
      val_(t) {
  }
  operator PyObject*() {
    return (PyObject*) val_;
  }
  operator T() {
    return val_;
  }
  operator void*() {
    return val_;
  }

  bool operator==(void* v) const {
    return val_ == v;
  }

  bool operator!=(void* v) const {
    return val_ != v;
  }

  T operator->() {
    return val_;
  }
};

RegisterFrame::RegisterFrame(RegisterCode* rcode, PyObject* obj, const ObjVector& args, const ObjVector& kw) :
    code(rcode) {
  instructions_ = code->instructions.data();
  current_hint = -1;

  if (rcode->function) {
    globals_ = PyFunction_GetGlobals(rcode->function);
    locals_ = NULL;
  } else {
    globals_ = PyEval_GetGlobals();
    locals_ = PyEval_GetGlobals();
  }

  Reg_Assert(kw.empty(), "Keyword args not supported.");

  builtins_ = PyEval_GetBuiltins();

  names_ = code->names();
  consts_ = code->consts();

#if ! STACK_ALLOC_REGISTERS
  registers = new PyObject*[rcode->num_registers];
#endif

  const int num_args = args.size();
  if (rcode->num_cells > 0) {
#if ! STACK_ALLOC_REGISTERS
    freevars = new PyObject*[rcode->num_cells];
#endif
    int i;
    for (i = 0; i < rcode->num_cellvars; ++i) {
      bool found_argname = false;
      char *cellname = PyString_AS_STRING(PyTuple_GET_ITEM(rcode->code()->co_cellvars, i));
      for (int arg_idx = 0; arg_idx < num_args; ++arg_idx) {
        char* argname = PyString_AS_STRING(PyTuple_GET_ITEM(rcode->code()->co_varnames, arg_idx));
        if (strcmp(cellname, argname) == 0) {
          PyObject* arg_value = args[arg_idx];
          freevars[i] = PyCell_New(arg_value);
          found_argname = true;
          break;
        }
      }
      if (!found_argname) {
        freevars[i] = PyCell_New(NULL);
      }
    }

    PyObject* closure = ((PyFunctionObject*) rcode->function)->func_closure;
    if (closure) {
      for (int i = rcode->num_cellvars; i < rcode->num_cells; ++i) {
        freevars[i] = PyTuple_GET_ITEM(closure, i - rcode->num_cellvars) ;
        Py_INCREF(freevars[i]);
      }
    } else {
      for (int i = rcode->num_cellvars; i < rcode->num_cells; ++i) {
        freevars[i] = PyCell_New(NULL);
      }
    }
  } else {
#if ! STACK_ALLOC_REGISTERS
    freevars = NULL;
#endif
  }

//  Log_Info("Alignments: reg: %d code: %d consts: %d globals: %d, this: %d",
//           ((long)registers) % 64, ((long)rcode->instructions.data()) % 64, (long)consts_ % 64, (long)globals_ % 64, (long)this % 64);

  const int num_registers = code->num_registers;

  // setup const and local register aliases.
  int num_consts = PyTuple_GET_SIZE(consts());
  for (int i = 0; i < num_consts; ++i) {
    registers[i] = PyTuple_GET_ITEM(consts(), i) ;
  }

  int needed_args = code->code()->co_argcount;
  int offset = num_consts;
  if (PyMethod_Check(obj)) {
    PyObject* self = PyMethod_GET_SELF(obj);

    Reg_Assert(self != NULL, "Method call without a bound self.");
    registers[offset++] = self;
    Py_INCREF(self);

    needed_args--;
  }

  if (code->function) {
    PyObject* def_args = PyFunction_GET_DEFAULTS(code->function);
    int num_def_args = def_args == NULL ? 0 : PyTuple_GET_SIZE(def_args);
{'fraction_non_alphanumeric': 0.14563628546861565, 'fraction_numerical': 0.004729148753224419, 'mean_word_length': 3.532391622016561, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2087691', 'n_tokens_mistral': 3433, 'n_tokens_neox': 3393, 'n_words': 752}