<commit_before>ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
require "mkmf"

ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
LIBSTEMMER = File.join(ROOT, 'libstemmer_c')

# build libstemmer_c
# FreeBSD make is gmake
make= (RUBY_PLATFORM =~ /freebsd/)? 'gmake' : 'make'
# MacOS architecture mess up
if RUBY_PLATFORM =~ /darwin/
  begin
    ENV['ARCHFLAGS']= "-arch " + %x[file #{File.expand_path(File.join(Config::CONFIG['bindir'], Config::CONFIG['RUBY_INSTALL_NAME']))}].strip!.match(/executable (.+)$/)[1] unless ENV['ARCHFLAGS'].nil?
  rescue
    $stderr << "Failed to get your ruby executable architecture.\n"
    $stderr << "Please specify one using $ARCHFLAGS environment variable.\n"
    exit
  end
end

# make libstemmer_c. unless we're cross-compiling.
unless RUBY_PLATFORM =~ /i386-mingw32/
  system "cd #{LIBSTEMMER}; #{make} libstemmer.o; cd #{ROOT};"
  exit unless $? == 0
end

$CFLAGS  += " -I#{File.expand_path(File.join(LIBSTEMMER, 'include'))} "
$libs    += " -L#{LIBSTEMMER} #{File.expand_path(File.join(LIBSTEMMER, 'libstemmer.o'))} "

if have_header("libstemmer.h")
  create_makefile("lingua/stemmer_native")
end<commit_msg>set COMMAND_MODE to unix2003 on MacOS (see #issue/9)<commit_after>ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
require "mkmf"

ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
LIBSTEMMER = File.join(ROOT, 'libstemmer_c')

# build libstemmer_c
# FreeBSD make is gmake
make= (RUBY_PLATFORM =~ /freebsd/)? 'gmake' : 'make'

# MacOS architecture mess up
if RUBY_PLATFORM =~ /darwin/
  # see: #issue/3, #issue/5
  begin
    ENV['ARCHFLAGS']= "-arch " + %x[file #{File.expand_path(File.join(Config::CONFIG['bindir'], Config::CONFIG['RUBY_INSTALL_NAME']))}].strip!.match(/executable (.+)$/)[1] unless ENV['ARCHFLAGS'].nil?
  rescue
    $stderr << "Failed to get your ruby executable architecture.\n"
    $stderr << "Please specify one using $ARCHFLAGS environment variable.\n"
    exit
  end
  # see: #issue/9, #issue/6
  # see: man compat
  if ENV['COMMAND_MODE'] == 'legacy'
    $stdout << "Setting compat mode to unix2003\n."
    ENV['COMMAND_MODE']= 'unix2003'
  end
end

# make libstemmer_c. unless we're cross-compiling.
unless RUBY_PLATFORM =~ /i386-mingw32/
  system "cd #{LIBSTEMMER}; #{make} libstemmer.o; cd #{ROOT};"
  exit unless $? == 0
end

$CFLAGS  += " -I#{File.expand_path(File.join(LIBSTEMMER, 'include'))} "
$libs    += " -L#{LIBSTEMMER} #{File.expand_path(File.join(LIBSTEMMER, 'libstemmer.o'))} "

if have_header("libstemmer.h")
  create_makefile("lingua/stemmer_native")
end
{'fraction_non_alphanumeric': 0.17935833011209895, 'fraction_numerical': 0.01198299188248937, 'mean_word_length': 4.776785714285714, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5769123', 'n_tokens_mistral': 1046, 'n_tokens_neox': 966, 'n_words': 203}