<commit_before>from cffi import FFI
ffi = FFI()
ffi.cdef("""
typedef uintptr_t VALUE;
typedef uintptr_t ID;

void ruby_init(void);
void ruby_init_loadpath(void);
void ruby_script(const char*);
VALUE rb_eval_string(const char*);
VALUE rb_protect(VALUE (*)(VALUE), VALUE, int*);
VALUE rb_funcall(VALUE, ID, int, ...);
VALUE rb_funcall2(VALUE, ID, int, const VALUE*);
ID rb_intern(const char*);
char *rb_string_value_ptr(volatile VALUE*);
VALUE rb_ivar_get(VALUE, ID);
VALUE rb_gv_get(const char*);
static inline int rb_type(VALUE obj);
int rb_respond_to(VALUE, ID);
VALUE rb_require(const char*);
VALUE rb_equal(VALUE,VALUE);
void rb_gc_register_address(VALUE*);
void rb_gc_unregister_address(VALUE*);
VALUE rb_proc_new(VALUE (*)(), VALUE);

extern VALUE rb_cObject;
extern VALUE rb_mKernel;

#define T_NONE   ...
#define T_NIL    ...
#define T_OBJECT ...
#define T_CLASS  ...
#define T_ICLASS ...
#define T_MODULE ...
#define T_FLOAT  ...
#define T_STRING ...
#define T_REGEXP ...
#define T_ARRAY  ...
#define T_HASH   ...
#define T_STRUCT ...
#define T_BIGNUM ...
#define T_FILE   ...
#define T_FIXNUM ...
#define T_TRUE   ...
#define T_FALSE  ...
#define T_DATA   ...
#define T_MATCH  ...
#define T_SYMBOL ...
#define T_RATIONAL ...
#define T_COMPLEX ...
#define T_UNDEF  ...
#define T_NODE   ...
#define T_ZOMBIE ...
#define T_MASK   ...

#define Qfalse ...
#define Qtrue  ...
#define Qnil   ...

long FIX2LONG_f(VALUE x);
long rb_str_strlen(VALUE);

char *RSTRING_PTR_f(VALUE x);
size_t RSTRING_LEN_f(VALUE x);
double RFLOAT_VALUE_f(VALUE x);

VALUE rb_sym_to_s(VALUE);

typedef struct {
	size_t len;
	VALUE *ptr;
} arr_len_and_ptr;
arr_len_and_ptr arr_len_and_ptr_f(VALUE x);

VALUE INT2NUM_f(int x);
VALUE rb_float_new(double);
VALUE rb_str_new(const char*, long);
VALUE rb_ary_new4(long, const VALUE *);
VALUE rb_hash_new(void);
VALUE rb_hash_aset(VALUE, VALUE, VALUE);
VALUE rb_hash_delete(VALUE,VALUE);
VALUE rb_sym_new(char *s, long len);

VALUE rb_const_get(VALUE, ID);

VALUE safe_rb_funcall2(VALUE obj, ID func, int argc, const VALUE *argv, int *error);
VALUE safe_rb_const_get(VALUE obj, ID name, int *error);
VALUE safe_rb_require(char *mod, int *error);

void *ruby_options(int, char**);
""")

C = ffi.verify("""
#include <ruby.h>

long FIX2LONG_f(VALUE x) {
	return FIX2LONG(x);
}

char *RSTRING_PTR_f(VALUE x) {
	return RSTRING_PTR(x);
}

size_t RSTRING_LEN_f(VALUE x) {
	return RSTRING_LEN(x);
}

double RFLOAT_VALUE_f(VALUE x) {
	return RFLOAT_VALUE(x);
}

typedef struct {
	size_t len;
	VALUE *ptr;
} arr_len_and_ptr;

arr_len_and_ptr arr_len_and_ptr_f(VALUE x) {
	return (arr_len_and_ptr){RARRAY_LEN(x), RARRAY_PTR(x)};
}<commit_msg>Clean up the ffi imports.<commit_after>from cffi import FFI
ffi = FFI()
ffi.cdef("""
typedef uintptr_t VALUE;
typedef uintptr_t ID;

// Initialisation.
void ruby_init(void);
void ruby_script(const char*);
void *ruby_options(int, char**);

// For testing.
VALUE rb_eval_string(const char*);
VALUE rb_equal(VALUE,VALUE);

extern VALUE rb_cObject;
void rb_gc_register_address(VALUE*);
void rb_gc_unregister_address(VALUE*);
VALUE rb_gv_get(const char*);
ID rb_intern(const char*);

static inline int rb_type(VALUE obj);
VALUE rb_proc_new(VALUE (*)(), VALUE);


#define T_NONE   ...
#define T_NIL    ...
#define T_OBJECT ...
#define T_CLASS  ...
#define T_ICLASS ...
#define T_MODULE ...
#define T_FLOAT  ...
#define T_STRING ...
#define T_REGEXP ...
#define T_ARRAY  ...
#define T_HASH   ...
#define T_STRUCT ...
#define T_BIGNUM ...
#define T_FILE   ...
#define T_FIXNUM ...
#define T_TRUE   ...
#define T_FALSE  ...
#define T_DATA   ...
#define T_MATCH  ...
#define T_SYMBOL ...
#define T_RATIONAL ...
#define T_COMPLEX ...
#define T_UNDEF  ...
#define T_NODE   ...
#define T_ZOMBIE ...
#define T_MASK   ...

#define Qfalse ...
#define Qtrue  ...
#define Qnil   ...

long FIX2LONG_f(VALUE x);

char *RSTRING_PTR_f(VALUE x);
size_t RSTRING_LEN_f(VALUE x);
double RFLOAT_VALUE_f(VALUE x);

VALUE rb_sym_to_s(VALUE);

typedef struct {
	size_t len;
	VALUE *ptr;
} arr_len_and_ptr;
arr_len_and_ptr arr_len_and_ptr_f(VALUE x);

VALUE INT2NUM_f(int x);
VALUE rb_float_new(double);
VALUE rb_str_new(const char*, long);
VALUE rb_ary_new4(long, const VALUE *);
VALUE rb_hash_new(void);
VALUE rb_hash_aset(VALUE, VALUE, VALUE);
VALUE rb_sym_new(char *s, long len);

// Safe wrappers.
VALUE safe_rb_funcall2(VALUE obj, ID func, int argc, const VALUE *argv, int *error);
VALUE safe_rb_const_get(VALUE obj, ID name, int *error);
VALUE safe_rb_require(char *mod, int *error);
""")

C = ffi.verify("""
#include <ruby.h>

long FIX2LONG_f(VALUE x) {
	return FIX2LONG(x);
}

char *RSTRING_PTR_f(VALUE x) {
	return RSTRING_PTR(x);
}

size_t RSTRING_LEN_f(VALUE x) {
	return RSTRING_LEN(x);
}

double RFLOAT_VALUE_f(VALUE x) {
	return RFLOAT_VALUE(x);
}

typedef struct {
	size_t len;
	VALUE *ptr;
} arr_len_and_ptr;

arr_len_and_ptr arr_len_and_ptr_f(VALUE x) {
	return (arr_len_and_ptr){RARRAY_LEN(x), RARRAY_PTR(x)};
}
{'fraction_non_alphanumeric': 0.14233797698364628, 'fraction_numerical': 0.0026246719160104987, 'mean_word_length': 4.648802736602052, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2425178', 'n_tokens_mistral': 2009, 'n_tokens_neox': 1971, 'n_words': 476}