<commit_before>require 'mkmf'
$CFLAGS += " -std=c99"

if have_library('xml2', 'xmlNewDoc') 
  # libxml2 libraries from http://www.xmlsoft.org/
  pkg_config('libxml-2.0')

  # nokogiri configuration from gem install
  nokogiri_lib = Gem.find_files('nokogiri').
    select { |name| name.match(%r{gems/nokogiri-([\d.]+)/lib/nokogiri}) }.
    sort_by {|name| name[/nokogiri-([\d.]+)/,1].split('.').map(&:to_i)}.last
  if nokogiri_lib
    nokogiri_ext = nokogiri_lib.sub(%r(lib/nokogiri(.rb)?$), 'ext/nokogiri')

    # if that doesn't work, try workarounds found in Nokogiri's extconf
    unless find_header('nokogiri.h', nokogiri_ext)
      require "#{nokogiri_ext}/extconf.rb"
    end

    # if found, enable direct calls to Nokogiri (and libxml2)
    $CFLAGS += ' -DNGLIB' if find_header('nokogiri.h', nokogiri_ext)
  end
end

# Symlink gumbo-parser source files.
rakehome = File.expand_path('../..')
unless File.exist? "#{rakehome}/ext/nokogumboc/gumbo.h"
  require 'fileutils'
  FileUtils.ln_s(Dir["#{rakehome}/gumbo-parser/src/*"],
                 "#{rakehome}/ext/nokogumboc", force: true)
  case RbConfig::CONFIG['target_os']
  when 'mingw32', /mswin/
    FileUtils.ln_s(Dir["#{rakehome}/gumbo-parser/visualc/include/*"],
                 "#{rakehome}/ext/nokogumboc", force: true)
  end
  # Set these to nil so that create_makefile picks up the new sources.
  $srcs = $objs = nil
end<commit_msg>Use File.join to construct paths<commit_after>require 'mkmf'
$CFLAGS += " -std=c99"

if have_library('xml2', 'xmlNewDoc') 
  # libxml2 libraries from http://www.xmlsoft.org/
  pkg_config('libxml-2.0')

  # nokogiri configuration from gem install
  nokogiri_lib = Gem.find_files('nokogiri').
    select { |name| name.match(%r{gems/nokogiri-([\d.]+)/lib/nokogiri}) }.
    sort_by {|name| name[/nokogiri-([\d.]+)/,1].split('.').map(&:to_i)}.last
  if nokogiri_lib
    nokogiri_ext = nokogiri_lib.sub(%r(lib/nokogiri(.rb)?$), 'ext/nokogiri')

    # if that doesn't work, try workarounds found in Nokogiri's extconf
    unless find_header('nokogiri.h', nokogiri_ext)
      require "#{nokogiri_ext}/extconf.rb"
    end

    # if found, enable direct calls to Nokogiri (and libxml2)
    $CFLAGS += ' -DNGLIB' if find_header('nokogiri.h', nokogiri_ext)
  end
end

# Symlink gumbo-parser source files.
ext_dir = File.dirname(__FILE__)
unless File.exist?(File.join(ext_dir, "gumbo.h"))
  require 'fileutils'
  gumbo_dir = File.expand_path('../../gumbo-parser', ext_dir)
  FileUtils.ln_s(Dir[File.join(gumbo_dir, 'src/*.[hc]')], ext_dir, force:true)
  case RbConfig::CONFIG['target_os']
  when 'mingw32', /mswin/
    FileUtils.ln_s(Dir[File.join(gumbo_dir, 'visualc/include/*.h')], ext_dir,
                   force: true)
  end
  # Set these to nil so that create_makefile picks up the new sources.
  $srcs = $objs = nil
end
{'fraction_non_alphanumeric': 0.1656005685856432, 'fraction_numerical': 0.007107320540156361, 'mean_word_length': 3.8450946643717727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4300370', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1101, 'n_words': 213}