<commit_before>#include "v8u.hpp"
#include "version.hpp"
#include <node_buffer.h>

#include <string>

extern "C" {
  #include"markdown.h"
  #include"html.h"
}

using namespace std;

using namespace node;
using namespace v8;
using namespace v8u;
//using namespace v8u::Version

namespace robotskirt {

// Constants taken from the official Sundown executable
#define OUTPUT_UNIT 64
#define DEFAULT_MAX_NESTING 16

////////////////////////////////////////////////////////////////////////////////
// UTILITIES to ease wrapping and interfacing with V8
////////////////////////////////////////////////////////////////////////////////

// Credit: @samcday
// http://sambro.is-super-awesome.com/2011/03/03/creating-a-proper-buffer-in-a-node-c-addon/ 
#define MAKE_FAST_BUFFER(NG_SLOW_BUFFER, NG_FAST_BUFFER)      \
  Local<Function> NG_JS_BUFFER = Local<Function>::Cast(       \
    Context::GetCurrent()->Global()->Get(                     \
      String::New("Buffer")));                                \
                                                              \
  Handle<Value> NG_JS_ARGS[2] = {                             \
    NG_SLOW_BUFFER,                                           \
    Integer::New(Buffer::Length(NG_SLOW_BUFFER))/*,           \
    Integer::New(0) <- WITH THIS WILL THROW AN ERROR*/        \
  };                                                          \
                                                              \
  NG_FAST_BUFFER = NG_JS_BUFFER->NewInstance(2, NG_JS_ARGS);

//DEPRECATED: Use Int() or Uint()
inline int64_t CheckInt(Handle<Value> value) {
  if (!value->IsInt32()) V8_THROW(TypeErr("You must provide an integer!"));
  return value->IntegerValue();
}

int CheckFlags(Handle<Value> hdl) {
  HandleScope scope;
  if (hdl->IsArray()) {
    int ret = 0;
    Handle<Array> array = Handle<Array>::Cast(hdl);
    for (uint32_t i=0; i<array->Length(); i++)
      ret |= Int(array->Get(i));
    return ret;
  }
  return Int(hdl);
}



////////////////////////////////////////////////////////////////////////////////
// RENDER FUNCTIONS STUFF
////////////////////////////////////////////////////////////////////////////////

// SIGNATURES
enum CppSignature {
    void_BUF1,
    void_BUF2,
    void_BUF2INT,
    void_BUF3,
     int_BUF1,
     int_BUF2,
     int_BUF2INT,
     int_BUF4
};

// A C++ WRAPPER FOR Buf*, to ensure deallocation

class BufWrap {
public:
    BufWrap(buf* buf): buf_(buf) {}
    ~BufWrap() {
        bufrelease(buf_);
    }
    buf* get() {return buf_;}
    buf* operator->() {return buf_;}
    buf* operator*() {return buf_;}
private:
    buf* const buf_;
};

// CONVERTERS (especially buf* to Local<Object>)

//DEPRECATED: use toString instead
Local<Object> toBuffer(const buf* buf) {
    HandleScope scope;
    Local<Object> ret;
    Handle<Object> buffer = Buffer::New(String::New((char*)buf->data, buf->size));
    MAKE_FAST_BUFFER(buffer, ret);
    return scope.Close(ret);
}
void setToBuf(buf* target, Handle<Object> obj) { //FIXME: test, this can lead to memory leaks
    bufreset(target);
    target->data = (uint8_t*)Buffer::Data(obj);
    target->asize = target->size = Buffer::Length(obj);
    Buffer::Initialize(obj);
}
inline void putToBuf(buf* target, Handle<Value> obj) {
    String::Utf8Value str (obj);
    bufput(target, *str, str.length());
}
inline Local<String> toString(const buf* buf) {
    return String::New(reinterpret_cast<const char*>(buf->data), buf->size);
}

// FUNCTION DATA (this gets injected into CPP functions converted to JS)
class FunctionData: public ObjectWrap {
public:
    FunctionData(void* function, CppSignature signature, void* opaque):
            function_(function), signature_(signature), opaque_(opaque) {
    }
    ~FunctionData() {}
    void* getFunction() {return function_;}
    void* getOpaque() {return opaque_;}
    CppSignature getSignature() {return signature_;}
    V8_CL_CALLBACK(FunctionData, ToString, 0) {
        return scope.Close(Str("<Native function>"));
    } V8_WRAP_END()
protected://TODO: add a shared_ptr to avoid opaque data deallocation
    void* const function_;
    CppSignature const signature_;
    void* const opaque_;
friend void setCppFunction(void*& func, Handle<Object> obj, CppSignature sig);
friend void jsFunction(Persistent<Object>& handle, void* func, CppSignature sig, InvocationCallback wrapper, void* opaque);
};

// CONVERT BETWEEN CPP AND JS FUNCTIONS

bool setCppFunction(void** func, void** opaque, Handle<Object> obj, CppSignature sig) {
    HandleScope scope;
    if (obj->InternalFieldCount() < 1) return false;
    if (obj->GetPointerFromInternalField(0) == NULL) return false; //FIXME
    FunctionData* data = ObjectWrap::Unwrap<FunctionData>(obj);
    if (sig != data->getSignature()) return false;
    *func = data->getFunction();
    *opaque = data->getOpaque();
    return true;
}

void jsFunction(Persistent<Object>& handle, void* func, CppSignature sig, InvocationCallback wrapper, void* opaque) {
    HandleScope scope;
    Local<ObjectTemplate> wrap = ObjectTemplate::New();
    wrap->SetInternalFieldCount(1);
    Local<FunctionTemplate> repr = Func(FunctionData::ToString);
    wrap->Set(Symbol("toString"), repr);
    wrap->Set(Symbol("inspect"), repr);
    wrap->SetCallAsFunctionHandler(wrapper);
    SetPersistent(handle, wrap->NewInstance());
    (new FunctionData(func,sig,opaque))->Wrap(handle);
}

//What to do if no function set?
#define NULL_ACTION()                                                          \
    V8_THROW(TypeErr("No function was set for this action."));

#define NULL_ACTION_R() NULL_ACTION()

// WRAPPERS (call a [wrapped] CPP function from JS)

#define WRAPPER_CALL_void()
#define WRAPPER_CALL_int() int v =

#define WRAPPER_POST_CALL_void()
#define WRAPPER_POST_CALL_int()                                                \
    if (!v) return scope.Close(False());

#define BUF1_WRAPPER(CPPFUNC, RET)                                             \
    static V8_CALLBACK(CPPFUNC##_wrapper, 0) {                                 \
        V8_UNWRAP(FunctionData, args)                                          \
                                                                               \
        BufWrap ob (bufnew(OUTPUT_UNIT));                                      \
        WRAPPER_CALL_##RET() ((RET(*)(buf*, void*))inst->getFunction())        \
                (*ob,  inst->getOpaque());                                     \
        WRAPPER_POST_CALL_##RET()                                              \
        return scope.Close(toString(*ob));                                     \
    } V8_WRAP_END()

// BINDERS (call a JS function from CPP)

#define BINDER_RETURN_void
#define BINDER_RETURN_int  1

#define BINDER_RETURN_NULL_void
#define BINDER_RETURN_NULL_int  1

#define BUF1_BINDER(CPPFUNC, RET)                                              \
    static RET CPPFUNC##_binder(struct buf *ob, void *opaque) {                \
        HandleScope scope;                                                     \
                                                                               \
        Persistent<Object>& obj = ((RendererWrap*)opaque)->CPPFUNC;            \
        if (obj.IsEmpty()) {                                                   \
            NULL_ACTION()                                                      \
        }                                                                      \
                                                                               \
        /*Convert arguments*/                                                  \
        Local<Value> args [0];                                                 \
                                                                               \
        /*Call it!*/                                                           \
        TryCatch trycatch;                                                     \
        Local<Value> ret = obj->CallAsFunction(Context::GetCurrent()->Global(), 0, args);\
        if (trycatch.HasCaught())                                              \
            V8_THROW(trycatch.Exception());                                    \
        /*Convert the result back*/                                            \
        if (ret->IsFalse()) return BINDER_RETURN_NULL_##RET;                   \
        putToBuf(ob, ret);                                                     \
        return BINDER_RETURN_##RET;                                            \
    }

// FORWARDERS (forward a Sundown call to its original C++ renderer)

#define BUF1_FORWARDER(CPPFUNC, RET)                                           \
    static RET CPPFUNC##_forwarder(struct buf *ob, void *opaque) {             \
        RendererWrap* rend = (RendererWrap*)opaque;                            \
        return ((RET(*)(struct buf *ob, void *opaque))rend->CPPFUNC##_orig)(   \
                ob,                                                            \
                rend->CPPFUNC##_opaque);                                       \
    }

//TODO

////////////////////////////////////////////////////////////////////////////////
// RENDERER CLASS DECLARATION
////////////////////////////////////////////////////////////////////////////////

// JS ACCESSORS

#define _RENDFUNC_GETTER(CPPFUNC)                                              \
    static V8_GETTER(CPPFUNC##_getter) {                                       \
        V8_UNWRAP(RendererWrap, info)                                          \
        if (inst->CPPFUNC.IsEmpty()) return scope.Close(Undefined());          \
        return scope.Close(inst->CPPFUNC);                                     \<commit_msg>Implement binders for BUF2 and BUF2INT<commit_after>#include "v8u.hpp"
#include "version.hpp"
#include <node_buffer.h>

#include <string>

extern "C" {
  #include"markdown.h"
  #include"html.h"
}

using namespace std;

using namespace node;
using namespace v8;
using namespace v8u;
//using namespace v8u::Version

namespace robotskirt {

// Constants taken from the official Sundown executable
#define OUTPUT_UNIT 64
#define DEFAULT_MAX_NESTING 16

////////////////////////////////////////////////////////////////////////////////
// UTILITIES to ease wrapping and interfacing with V8
////////////////////////////////////////////////////////////////////////////////

// Credit: @samcday
// http://sambro.is-super-awesome.com/2011/03/03/creating-a-proper-buffer-in-a-node-c-addon/ 
#define MAKE_FAST_BUFFER(NG_SLOW_BUFFER, NG_FAST_BUFFER)      \
  Local<Function> NG_JS_BUFFER = Local<Function>::Cast(       \
    Context::GetCurrent()->Global()->Get(                     \
      String::New("Buffer")));                                \
                                                              \
  Handle<Value> NG_JS_ARGS[2] = {                             \
    NG_SLOW_BUFFER,                                           \
    Integer::New(Buffer::Length(NG_SLOW_BUFFER))/*,           \
    Integer::New(0) <- WITH THIS WILL THROW AN ERROR*/        \
  };                                                          \
                                                              \
  NG_FAST_BUFFER = NG_JS_BUFFER->NewInstance(2, NG_JS_ARGS);

//DEPRECATED: Use Int() or Uint()
inline int64_t CheckInt(Handle<Value> value) {
  if (!value->IsInt32()) V8_THROW(TypeErr("You must provide an integer!"));
  return value->IntegerValue();
}

int CheckFlags(Handle<Value> hdl) {
  HandleScope scope;
  if (hdl->IsArray()) {
    int ret = 0;
    Handle<Array> array = Handle<Array>::Cast(hdl);
    for (uint32_t i=0; i<array->Length(); i++)
      ret |= Int(array->Get(i));
    return ret;
  }
  return Int(hdl);
}



////////////////////////////////////////////////////////////////////////////////
// RENDER FUNCTIONS STUFF
////////////////////////////////////////////////////////////////////////////////

// SIGNATURES
enum CppSignature {
    void_BUF1,
    void_BUF2,
    void_BUF2INT,
    void_BUF3,
     int_BUF1,
     int_BUF2,
     int_BUF2INT,
     int_BUF4
};

// A C++ WRAPPER FOR Buf*, to ensure deallocation

class BufWrap {
public:
    BufWrap(buf* buf): buf_(buf) {}
    ~BufWrap() {
        bufrelease(buf_);
    }
    buf* get() {return buf_;}
    buf* operator->() {return buf_;}
    buf* operator*() {return buf_;}
private:
    buf* const buf_;
};

// CONVERTERS (especially buf* to Local<Object>)

//DEPRECATED: use toString instead
Local<Object> toBuffer(const buf* buf) {
    HandleScope scope;
    Local<Object> ret;
    Handle<Object> buffer = Buffer::New(String::New((char*)buf->data, buf->size));
    MAKE_FAST_BUFFER(buffer, ret);
    return scope.Close(ret);
}
void setToBuf(buf* target, Handle<Object> obj) { //FIXME: test, this can lead to memory leaks
    bufreset(target);
    target->data = (uint8_t*)Buffer::Data(obj);
    target->asize = target->size = Buffer::Length(obj);
    Buffer::Initialize(obj);
}
inline void putToBuf(buf* target, Handle<Value> obj) {
    String::Utf8Value str (obj);
    bufput(target, *str, str.length());
}
inline Local<String> toString(const buf* buf) {
    return String::New(reinterpret_cast<const char*>(buf->data), buf->size);
}

// FUNCTION DATA (this gets injected into CPP functions converted to JS)
class FunctionData: public ObjectWrap {
public:
    FunctionData(void* function, CppSignature signature, void* opaque):
            function_(function), signature_(signature), opaque_(opaque) {
    }
    ~FunctionData() {}
    void* getFunction() {return function_;}
    void* getOpaque() {return opaque_;}
    CppSignature getSignature() {return signature_;}
    V8_CL_CALLBACK(FunctionData, ToString, 0) {
        return scope.Close(Str("<Native function>"));
    } V8_WRAP_END()
protected://TODO: add a shared_ptr to avoid opaque data deallocation
    void* const function_;
    CppSignature const signature_;
    void* const opaque_;
friend void setCppFunction(void*& func, Handle<Object> obj, CppSignature sig);
friend void jsFunction(Persistent<Object>& handle, void* func, CppSignature sig, InvocationCallback wrapper, void* opaque);
};

// CONVERT BETWEEN CPP AND JS FUNCTIONS

bool setCppFunction(void** func, void** opaque, Handle<Object> obj, CppSignature sig) {
    HandleScope scope;
    if (obj->InternalFieldCount() < 1) return false;
    if (obj->GetPointerFromInternalField(0) == NULL) return false; //FIXME
    FunctionData* data = ObjectWrap::Unwrap<FunctionData>(obj);
    if (sig != data->getSignature()) return false;
    *func = data->getFunction();
    *opaque = data->getOpaque();
    return true;
}

void jsFunction(Persistent<Object>& handle, void* func, CppSignature sig, InvocationCallback wrapper, void* opaque) {
    HandleScope scope;
    Local<ObjectTemplate> wrap = ObjectTemplate::New();
    wrap->SetInternalFieldCount(1);
    Local<FunctionTemplate> repr = Func(FunctionData::ToString);
    wrap->Set(Symbol("toString"), repr);
    wrap->Set(Symbol("inspect"), repr);
    wrap->SetCallAsFunctionHandler(wrapper);
    SetPersistent(handle, wrap->NewInstance());
    (new FunctionData(func,sig,opaque))->Wrap(handle);
}

//What to do if no function set?
#define NULL_ACTION()                                                          \
    V8_THROW(TypeErr("No function was set for this action."));

#define NULL_ACTION_R() NULL_ACTION()

// WRAPPERS (call a [wrapped] CPP function from JS)

#define WRAPPER_CALL_void()
#define WRAPPER_CALL_int() int v =

#define WRAPPER_POST_CALL_void()
#define WRAPPER_POST_CALL_int()                                                \
    if (!v) return scope.Close(False());

#define BUF1_WRAPPER(CPPFUNC, RET)                                             \
    static V8_CALLBACK(CPPFUNC##_wrapper, 0) {                                 \
        V8_UNWRAP(FunctionData, args)                                          \
                                                                               \
        BufWrap ob (bufnew(OUTPUT_UNIT));                                      \
        WRAPPER_CALL_##RET() ((RET(*)(buf*, void*))inst->getFunction())        \
                (*ob,  inst->getOpaque());                                     \
        WRAPPER_POST_CALL_##RET()                                              \
        return scope.Close(toString(*ob));                                     \
    } V8_WRAP_END()

// BINDERS (call a JS function from CPP)

#define BINDER_RETURN_void
#define BINDER_RETURN_int  1

#define BINDER_RETURN_NULL_void
#define BINDER_RETURN_NULL_int  1

#define BUF1_BINDER(CPPFUNC, RET)                                              \
    static RET CPPFUNC##_binder(struct buf *ob, void *opaque) {                \
        HandleScope scope;                                                     \
                                                                               \
        Persistent<Object>& obj = ((RendererWrap*)opaque)->CPPFUNC;            \
        if (obj.IsEmpty()) {                                                   \
            NULL_ACTION()                                                      \
        }                                                                      \
                                                                               \
        /*Convert arguments*/                                                  \
        Local<Value> args [0];                                                 \
                                                                               \
        /*Call it!*/                                                           \
        TryCatch trycatch;                                                     \
        Local<Value> ret = obj->CallAsFunction(Context::GetCurrent()->Global(), 0, args);\
        if (trycatch.HasCaught())                                              \
            V8_THROW(trycatch.Exception());                                    \
        /*Convert the result back*/                                            \
        if (ret->IsFalse()) return BINDER_RETURN_NULL_##RET;                   \
        putToBuf(ob, ret);                                                     \
        return BINDER_RETURN_##RET;                                            \
    }

#define BUF2_BINDER(CPPFUNC, RET)                                              \
    static RET CPPFUNC##_binder(struct buf *ob, const struct buf *text, void *opaque) {\
        HandleScope scope;                                                     \
                                                                               \
        Persistent<Object>& obj = ((RendererWrap*)opaque)->CPPFUNC;            \
        if (obj.IsEmpty()) {                                                   \
            NULL_ACTION()                                                      \
        }                                                                      \
                                                                               \
        /*Convert arguments*/                                                  \
        Local<Value> args [1] = {toString(text)};                              \
                                                                               \
        /*Call it!*/                                                           \
        TryCatch trycatch;                                                     \
        Local<Value> ret = obj->CallAsFunction(Context::GetCurrent()->Global(), 1, args);\
        if (trycatch.HasCaught())                                              \
            V8_THROW(trycatch.Exception());                                    \
        /*Convert the result back*/                                            \
        if (ret->IsFalse()) return BINDER_RETURN_NULL_##RET;                   \
        putToBuf(ob, ret);                                                     \
        return BINDER_RETURN_##RET;                                            \
    }

#define BUF2INT_BINDER(CPPFUNC, RET)                                           \
    static RET CPPFUNC##_binder(struct buf *ob, const struct buf *text, int flags, void *opaque) {\
        HandleScope scope;                                                     \
                                                                               \
        Persistent<Object>& obj = ((RendererWrap*)opaque)->CPPFUNC;            \
        if (obj.IsEmpty()) {                                                   \
            NULL_ACTION()                                                      \
        }                                                                      \
                                                                               \
        /*Convert arguments*/                                                  \
        Local<Value> args [2] = {toString(text), Int(flags)};                  \
                                                                               \
        /*Call it!*/                                                           \
        TryCatch trycatch;                                                     \
        Local<Value> ret = obj->CallAsFunction(Context::GetCurrent()->Global(), 2, args);\
        if (trycatch.HasCaught())                                              \
            V8_THROW(trycatch.Exception());                                    \
        /*Convert the result back*/                                            \
        if (ret->IsFalse()) return BINDER_RETURN_NULL_##RET;                   \
        putToBuf(ob, ret);                                                     \
        return BINDER_RETURN_##RET;                                            \
    }

// FORWARDERS (forward a Sundown call to its original C++ renderer)

#define BUF1_FORWARDER(CPPFUNC, RET)                                           \
    static RET CPPFUNC##_forwarder(struct buf *ob, void *opaque) {             \
        RendererWrap* rend = (RendererWrap*)opaque;                            \
        return ((RET(*)(struct buf *ob, void *opaque))rend->CPPFUNC##_orig)(   \
                ob,                                                            \
                rend->CPPFUNC##_opaque);                                       \
    }

//TODO

////////////////////////////////////////////////////////////////////////////////
// RENDERER CLASS DECLARATION
////////////////////////////////////////////////////////////////////////////////

// JS ACCESSORS

#define _RENDFUNC_GETTER(CPPFUNC)                                              \
    static V8_GETTER(CPPFUNC##_getter) {                                       \
        V8_UNWRAP(RendererWrap, info)                                          \
        if (inst->CPPFUNC.IsEmpty()) return scope.Close(Undefined());          \
        return scope.Close(inst->CPPFUNC);                                     \
{'fraction_non_alphanumeric': 0.14222066917096182, 'fraction_numerical': 0.005678343670830786, 'mean_word_length': 1.1465404087755484, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 147, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 212, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '610216', 'n_tokens_mistral': 5548, 'n_tokens_neox': 5388, 'n_words': 1226}