<commit_before>#This class reads .po-files generated by something like POEdit and can be used to run multi-language applications or websites.
class GettextSimple
  #Initializes various data.
  def initialize(args = {})
    @args = {
      :encoding => "utf-8",
      :i18n => false,
      :default_locale => "en"
    }.merge(args)
    @locales = {}
    
    # To avoid doing a hash lookup on every translate.
    @i18n = @args[:i18n]
    
    if @i18n
      @default_locale = I18n.default_locale
    else
      @default_locale = @args[:default_locale]
    end
    
    @locale = @default_locale
  end
  
  def register_kernel_methods
    require "#{__dir__}/../include/kernel_methods"
    $gettext_simple_kernel_instance = self
  end
  
  def locale_for_thread=(newlocale)
    Thread.current[:gettext_simple_locale] = newlocale
  end
  
  #Loads a 'locales'-directory with .mo- and .po-files and fills the '@locales'-hash.
  #===Examples
  # gtext.load_dir("#{File.dirname(__FILE__)}/../locales")
  def load_dir(dir)
    check_folders = ["LC_MESSAGES", "LC_ALL"]
    
    Dir.foreach(dir) do |file|
      fn = "#{dir}/#{file}"
      if File.directory?(fn) && file.match(/^[a-z]{2}/)
        @locales[file] = {} unless @locales[file]
        
        check_folders.each do |fname|
          fpath = "#{dir}/#{file}/#{fname}"
          
          if File.exists?(fpath) && File.directory?(fpath)
            Dir.foreach(fpath) do |pofile|
              if pofile.match(/\.po$/)
                pofn = "#{dir}/#{file}/#{fname}/#{pofile}"
                scan_pofile(file, pofn)
              end
            end
          end
        end
      end
    end
  end
  
  #Translates a given string to a given locale from the read .po-files.
  #===Examples
  # str = "Hello" #=> "Hello"
  # gtext.trans("da_DK", str) #=> "Hej"
  def translate_with_locale(locale, str, replaces = nil)
    locale = locale.to_s
    str = str.to_s
    raise "Locale was not found: '#{locale}' in '#{@locales.keys.join(", ")}'." unless @locales.key?(locale)
    
    if !@locales[locale].key?(str)
      translated_str = str
    else
      translated_str = @locales[locale][str]
    end
    
    if replaces
      replaces.each do |key, val|
        translated_str = translated_str.gsub("%{#{key}}", val)
      end
    end
    
    return translated_str
  end
  
  def translate(str, replaces = nil)
    if @i18n
      locale = I18n.locale
    elsif locale = Thread.current[:gettext_simple_locale]
      # Locale already set through condition.
    else
      locale = @default_locale
    end
    
    translate_with_locale(locale, str, replaces)
  end
  
  # Returns an array of available locales.
  def locales
    return @locales.keys
  end
  
private
  
  def scan_pofile(locale, filepath)
    current_id = nil
    current_translation = nil
    
    reading_id = false
    reading_translation = false
    
    File.open(filepath, "r", :encoding => @args[:encoding]) do |fp|
      fp.each_line do |line|
        if match = line.match(/^(msgid|msgstr)\s+"(.*)"$/)
          if match[1] == "msgid"
            current_id = match[2]
            reading_id = true
          elsif match[1] == "msgstr"
            current_translation = match[2]
            reading_id = false
            reading_translation = true
          else
            raise "Unknown translation parameter: '#{match[1]}'."
          end
        elsif match = line.match(/^"(.*)"$/)
          if reading_id
            current_id << match[1]
          elsif reading_translation
            current_translation << match[1]
          else
            raise "Text given but don't know what to do with it."
          end
        elsif line.match(/^(\r|)\n$/)
          if reading_id
            reading_id = false
          elsif reading_translation
            add_translation(locale, current_id, current_translation)
            reading_translation = false
            current_id = nil
            current_translation = nil
          end
        elsif line.start_with?("#: ")
          # Line is a comment explaining where the translation comes from - ignore.
        else
          raise "Couldn't understand line: '#{line}'."
        end
      end
      
      add_translation(locale, current_id, current_translation) if reading_translation
    end
    
    #cont.scan(/msgid\s+\"(.+)\"(\r|)\nmsgstr\s+\"(.+)\"(\r|)\n(\r|)\n/) do |match|
    #  @locales[locale][match[0]] = match[2].to_s.encode("utf-8")
    #end
  end
  
  def add_translation(locale, key, val)
    raise "No such language: '#{locale}'." unless @locales.key?(locale)
    @locales[locale][key] = val
  end
end
<commit_msg>Ignore all comments.<commit_after>#This class reads .po-files generated by something like POEdit and can be used to run multi-language applications or websites.
class GettextSimple
  #Initializes various data.
  def initialize(args = {})
    @args = {
      :encoding => "utf-8",
      :i18n => false,
      :default_locale => "en"
    }.merge(args)
    @locales = {}
    
    # To avoid doing a hash lookup on every translate.
    @i18n = @args[:i18n]
    
    if @i18n
      @default_locale = I18n.default_locale
    else
      @default_locale = @args[:default_locale]
    end
    
    @locale = @default_locale
  end
  
  def register_kernel_methods
    require "#{__dir__}/../include/kernel_methods"
    $gettext_simple_kernel_instance = self
  end
  
  def locale_for_thread=(newlocale)
    Thread.current[:gettext_simple_locale] = newlocale
  end
  
  #Loads a 'locales'-directory with .mo- and .po-files and fills the '@locales'-hash.
  #===Examples
  # gtext.load_dir("#{File.dirname(__FILE__)}/../locales")
  def load_dir(dir)
    check_folders = ["LC_MESSAGES", "LC_ALL"]
    
    Dir.foreach(dir) do |file|
      fn = "#{dir}/#{file}"
      if File.directory?(fn) && file.match(/^[a-z]{2}/)
        @locales[file] = {} unless @locales[file]
        
        check_folders.each do |fname|
          fpath = "#{dir}/#{file}/#{fname}"
          
          if File.exists?(fpath) && File.directory?(fpath)
            Dir.foreach(fpath) do |pofile|
              if pofile.match(/\.po$/)
                pofn = "#{dir}/#{file}/#{fname}/#{pofile}"
                scan_pofile(file, pofn)
              end
            end
          end
        end
      end
    end
  end
  
  #Translates a given string to a given locale from the read .po-files.
  #===Examples
  # str = "Hello" #=> "Hello"
  # gtext.trans("da_DK", str) #=> "Hej"
  def translate_with_locale(locale, str, replaces = nil)
    locale = locale.to_s
    str = str.to_s
    raise "Locale was not found: '#{locale}' in '#{@locales.keys.join(", ")}'." unless @locales.key?(locale)
    
    if !@locales[locale].key?(str)
      translated_str = str
    else
      translated_str = @locales[locale][str]
    end
    
    if replaces
      replaces.each do |key, val|
        translated_str = translated_str.gsub("%{#{key}}", val)
      end
    end
    
    return translated_str
  end
  
  def translate(str, replaces = nil)
    if @i18n
      locale = I18n.locale
    elsif locale = Thread.current[:gettext_simple_locale]
      # Locale already set through condition.
    else
      locale = @default_locale
    end
    
    translate_with_locale(locale, str, replaces)
  end
  
  # Returns an array of available locales.
  def locales
    return @locales.keys
  end
  
private
  
  def scan_pofile(locale, filepath)
    current_id = nil
    current_translation = nil
    
    reading_id = false
    reading_translation = false
    
    File.open(filepath, "r", :encoding => @args[:encoding]) do |fp|
      fp.each_line do |line|
        if match = line.match(/^(msgid|msgstr)\s+"(.*)"$/)
          if match[1] == "msgid"
            current_id = match[2]
            reading_id = true
          elsif match[1] == "msgstr"
            current_translation = match[2]
            reading_id = false
            reading_translation = true
          else
            raise "Unknown translation parameter: '#{match[1]}'."
          end
        elsif match = line.match(/^"(.*)"$/)
          if reading_id
            current_id << match[1]
          elsif reading_translation
            current_translation << match[1]
          else
            raise "Text given but don't know what to do with it."
          end
        elsif line.match(/^(\r|)\n$/)
          if reading_id
            reading_id = false
          elsif reading_translation
            add_translation(locale, current_id, current_translation)
            reading_translation = false
            current_id = nil
            current_translation = nil
          end
        elsif line.start_with?("#")
          # Line is a comment - ignore.
        else
          raise "Couldn't understand line: '#{line}'."
        end
      end
      
      add_translation(locale, current_id, current_translation) if reading_translation
    end
    
    #cont.scan(/msgid\s+\"(.+)\"(\r|)\nmsgstr\s+\"(.+)\"(\r|)\n(\r|)\n/) do |match|
    #  @locales[locale][match[0]] = match[2].to_s.encode("utf-8")
    #end
  end
  
  def add_translation(locale, key, val)
    raise "No such language: '#{locale}'." unless @locales.key?(locale)
    @locales[locale][key] = val
  end
end

{'fraction_non_alphanumeric': 0.12650537051101227, 'fraction_numerical': 0.005641748942172073, 'mean_word_length': 2.1536093055080396, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '196832', 'n_tokens_mistral': 2986, 'n_tokens_neox': 2943, 'n_words': 706}