<commit_before>require 'rubygems'
require 'bundler'
Bundler.require(:default)

class Tagelizer

  attr_reader :locale, :options
  def initialize(locale = 'en')
    @dictionary= (dictionaries.include?(locale) ? locale : "en")
    @minwordsize = 2
    @options         = {'ignore-case' => true}
  end

  def parse( text )
    text.split(" ").collect {|i| /(\w*)/.match(i)[1]}.select {|i| i.size > @minwordsize}.collect {|w| stemmer.stem corrected_word(w)}
  end

  def speller
    @speller ||= build_speller
  end

  def build_speller
    speller = Aspell.new(dictionary)
    speller.suggestion_mode = actual_suggestion_mode
    actual_options.each do |key, value|
      speller.set_option key, value
    end

    speller
  end

  def stemmer
    @stemmer ||= build_stemmer
  end

  def build_stemmer
    Lingua::Stemmer.new(:language => dictionary)
  end

  attr_reader :dictionary
  def dictionary=(dict)
    unless dictionaries.include?(dict)
      raise ArgumentError, 'unknown dictionary'
    end

    @dictionary = dict
  end

  def dictionaries
    @dictionaries ||= Aspell.list_dicts.collect { |dict| dict.code }
  end

  def corrected_word(word)
    speller.check(word) ? word : speller.suggest(word).first
  end

  def actual_options
    options.keys.inject({}) do |hash, key|
      hash[key] = options[key].to_s
      hash
    end
  end



end
<commit_msg>fixed dump error from refactoring<commit_after>require 'rubygems'
require 'bundler'
Bundler.require(:default)

class Tagelizer

  attr_reader :locale, :options
  def initialize(locale = 'en')
    @dictionary= (dictionaries.include?(locale) ? locale : "en")
    @minwordsize = 2
    @options         = {'ignore-case' => true}
  end

  def parse( text )
    text.split(" ").collect {|i| /(\w*)/.match(i)[1]}.select {|i| i.size > @minwordsize}.collect {|w| stemmer.stem corrected_word(w)}
  end

  def speller
    @speller ||= build_speller
  end

  def build_speller
    speller = Aspell.new(dictionary)
    speller.suggestion_mode = 'normal'
    actual_options.each do |key, value|
      speller.set_option key, value
    end

    speller
  end

  def stemmer
    @stemmer ||= build_stemmer
  end

  def build_stemmer
    Lingua::Stemmer.new(:language => dictionary)
  end

  attr_reader :dictionary
  def dictionary=(dict)
    unless dictionaries.include?(dict)
      raise ArgumentError, 'unknown dictionary'
    end

    @dictionary = dict
  end

  def dictionaries
    @dictionaries ||= Aspell.list_dicts.collect { |dict| dict.code }
  end

  def corrected_word(word)
    speller.check(word) ? word : speller.suggest(word).first
  end

  def actual_options
    options.keys.inject({}) do |hash, key|
      hash[key] = options[key].to_s
      hash
    end
  end



end

{'fraction_non_alphanumeric': 0.11695906432748537, 'fraction_numerical': 0.0014619883040935672, 'mean_word_length': 3.204301075268817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3899619', 'n_tokens_mistral': 967, 'n_tokens_neox': 917, 'n_words': 218}