<commit_before>#include <ruby.h>
#if defined(HAVE_RUBY_ENCODING_H)
#include <ruby/encoding.h>
#endif

#include "unf/normalizer.hh"

extern "C" {
  VALUE unf_allocate(VALUE klass);
  VALUE unf_initialize(VALUE self);
  VALUE unf_delete(UNF::Normalizer* ptr);
  VALUE unf_normalize(VALUE self, VALUE source, VALUE normalization_form);

  ID FORM_NFD;
  ID FORM_NFC;
  ID FORM_NFKD;
  ID FORM_NFKC;

  void Init_unf_ext() {
    VALUE mdl = rb_define_module("UNF");

    VALUE cls = rb_define_class_under(mdl, "Normalizer", rb_cObject);
    rb_define_alloc_func(cls, unf_allocate);
    rb_define_method(cls, "initialize", (VALUE (*)(...))unf_initialize, 0);
    rb_define_method(cls, "normalize", (VALUE (*)(...))unf_normalize, 2);<commit_msg>Include ruby.h after unf/normalizer.hh so fix build with 1.8 on win32.<commit_after>#include "unf/normalizer.hh"

#include <ruby.h>
#if defined(HAVE_RUBY_ENCODING_H)
#include <ruby/encoding.h>
#endif

extern "C" {
  VALUE unf_allocate(VALUE klass);
  VALUE unf_initialize(VALUE self);
  VALUE unf_delete(UNF::Normalizer* ptr);
  VALUE unf_normalize(VALUE self, VALUE source, VALUE normalization_form);

  ID FORM_NFD;
  ID FORM_NFC;
  ID FORM_NFKD;
  ID FORM_NFKC;

  void Init_unf_ext() {
    VALUE mdl = rb_define_module("UNF");

    VALUE cls = rb_define_class_under(mdl, "Normalizer", rb_cObject);
    rb_define_alloc_func(cls, unf_allocate);
    rb_define_method(cls, "initialize", (VALUE (*)(...))unf_initialize, 0);
    rb_define_method(cls, "normalize", (VALUE (*)(...))unf_normalize, 2);
{'fraction_non_alphanumeric': 0.13615635179153093, 'fraction_numerical': 0.005211726384364821, 'mean_word_length': 5.295081967213115, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024045', 'n_tokens_mistral': 622, 'n_tokens_neox': 576, 'n_words': 123}