<commit_before>require 'formula'

class MecabIpadic <Formula
  url 'http://downloads.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz'
  homepage 'http://mecab.sourceforge.net/'
  md5 'e09556657cc70e45564c6514a6b08495'

  depends_on "mecab"

  def install
    args = ["--disable-debug", "--disable-dependency-tracking", "--prefix=#{prefix}"]
    args << ARGV.find(Proc.new {"--with-charset=utf8"}) { |arg| /^--with-charset/.match(arg) }
    
    system "./configure", *args
    system "make install"
  end
end<commit_msg>mecab-ipadic - remove trailing whitespace<commit_after>require 'formula'

class MecabIpadic <Formula
  url 'http://downloads.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz'
  homepage 'http://mecab.sourceforge.net/'
  md5 'e09556657cc70e45564c6514a6b08495'

  depends_on "mecab"

  def install
    args = ["--disable-debug", "--disable-dependency-tracking", "--prefix=#{prefix}"]
    args << ARGV.find(Proc.new {"--with-charset=utf8"}) { |arg| /^--with-charset/.match(arg) }

    system "./configure", *args
    system "make install"
  end
end
{'fraction_non_alphanumeric': 0.17504332755632582, 'fraction_numerical': 0.08492201039861352, 'mean_word_length': 3.8326359832635983, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3589223', 'n_tokens_mistral': 521, 'n_tokens_neox': 444, 'n_words': 66}