<commit_before>require 'rbconfig'
bindir = RbConfig::CONFIG['bindir']
# autodetect ruby headers
if bindir =~ %r{(^.*/\.rbenv/versions)/([^/]+)/bin$}
  ruby_include = "#{$1}/#{$2}/include/ruby-1.9.1/ruby-#{$2}"
  ARGV << "--with-ruby-include=#{ruby_include}"
elsif bindir =~ %r{(^.*/\.rvm/rubies)/([^/]+)/bin$}
  ruby_include = "#{$1}/#{$2}/include/ruby-1.9.1/#{$2}"
  ARGV << "--with-ruby-include=#{ruby_include}"
end

require "mkmf"
require "debugger/ruby_core_source"
require 'fileutils'

if RUBY_VERSION < "1.9"
  STDERR.print("Ruby version is too old\n")
  exit(1)
end

hdrs = if RUBY_VERSION == '1.9.2'
  lambda {
    have_struct_member("rb_method_entry_t", "body", "method.h")
    have_header("vm_core.h") and have_header("iseq.h") and have_header("insns.inc") and
    have_header("insns_info.inc") and have_header("eval_intern.h")
  }<commit_msg>don't override user with-ruby-include arg<commit_after># autodetect ruby headers
unless ARGV.any? {|arg| arg.include?('--with-ruby-include') }
  require 'rbconfig'
  bindir = RbConfig::CONFIG['bindir']
  if bindir =~ %r{(^.*/\.rbenv/versions)/([^/]+)/bin$}
    ruby_include = "#{$1}/#{$2}/include/ruby-1.9.1/ruby-#{$2}"
    ARGV << "--with-ruby-include=#{ruby_include}"
  elsif bindir =~ %r{(^.*/\.rvm/rubies)/([^/]+)/bin$}
    ruby_include = "#{$1}/#{$2}/include/ruby-1.9.1/#{$2}"
    ARGV << "--with-ruby-include=#{ruby_include}"
  end
end

require "mkmf"
require "debugger/ruby_core_source"
require 'fileutils'

if RUBY_VERSION < "1.9"
  STDERR.print("Ruby version is too old\n")
  exit(1)
end

hdrs = if RUBY_VERSION == '1.9.2'
  lambda {
    have_struct_member("rb_method_entry_t", "body", "method.h")
    have_header("vm_core.h") and have_header("iseq.h") and have_header("insns.inc") and
    have_header("insns_info.inc") and have_header("eval_intern.h")
  }
{'fraction_non_alphanumeric': 0.22387238723872388, 'fraction_numerical': 0.019801980198019802, 'mean_word_length': 4.429850746268657, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4521029', 'n_tokens_mistral': 782, 'n_tokens_neox': 723, 'n_words': 120}