<commit_before># encoding: utf-8
# This file is part of the K5 bot project.
# See files README.md and COPYING for copyright and licensing information.

# Translate plugin

require_relative '../../IRCPlugin'
require 'rubygems'
require 'nokogiri'
require 'net/http'
require 'json'

class Translate < IRCPlugin
  Description = "Uses translation engines to translate between languages."
  Dependencies = [ :Language ]

  def self.to_lang_key(x)
    x.to_sym
  end

  def self.make_lang_service_format_map(verbatim_array, modifications_hash = {})
    result = {}
    verbatim_array.each do |x|
      result[to_lang_key(x)] = x
    end
    modifications_hash.each do |k, v|
      result[to_lang_key(k)] = v
    end
    result
  end

  def self.lang_to_service_format(l_from, l_to, possibles)
    return nil unless possibles.include? l_from # "Can't translate from #{l_from} with #{service}"
    return nil unless ((possibles.include? l_to) && !(:auto.eql? l_to)) # "Can't translate to #{l_to} with #{service}"
    [possibles[l_from], possibles[l_to]]
  end

  GOOGLE_SUPPORTED = make_lang_service_format_map(%w(auto en ja ko fr pt de it es no ru fi hu sv da pl lt), {:zh => 'zh-CN', :tw => 'zh-TW'})
  HONYAKU_SUPPORTED = make_lang_service_format_map(%w(en ja ko fr pt zh de it es))
  EXCITE_SUPPORTED = make_lang_service_format_map([], {:en => 'EN', :ja => 'JA'})
  KNOWN_SERVICES = {
      :Google => {:prefix=>'g', :languages=>GOOGLE_SUPPORTED, :translator=>:google_translate},
      :Honyaku => {:prefix=>'h', :languages=>HONYAKU_SUPPORTED, :translator=>:honyaku_translate},
      :Excite => {:prefix=>'x', :languages=>EXCITE_SUPPORTED, :translator=>:excite_translate},
  }
  DEFAULT_SERVICE = :Honyaku
  DEFAULT_SERVICE_LANGUAGES = {:en=>'e', :ja=>'j', :ko=>'k', :zh=>'c'}

  # Internal unified language id =>
  # [Shortcut form for commands, Language description for help]
  LANGUAGE_INFO = {
      :auto => ['_', 'Auto-detected language'],
      :en => ['en', 'English'],
      :ja => ['ja', 'Japanese'],
      :zh => ['zh', 'Simplified Chinese'],
      :tw => ['tw', 'Traditional Chinese'],
      :ko => ['ko', 'Korean'],
      :fr => ['fr', 'French'],
      :pt => ['pt', 'Portuguese'],
      :de => ['de', 'German'],
      :it => ['it', 'Italian'],
      :es => ['es', 'Spanish'],
      :no => ['no', 'Norwegian'],
      :ru => ['ru', 'Russian'],
      :fi => ['fi', 'Finnish'],
      :hu => ['hu', 'Hungarian'],
      :sv => ['sv', 'Swedish'],
      :da => ['da', 'Danish'],
      :pl => ['pl', 'Polish'],
      :lt => ['lt', 'Lithuanian'],
  }

  def self.get_language_info(lang)
    result = LANGUAGE_INFO[lang]
    raise "Cannot find language info for #{lang}" unless (result && (result.instance_of? Array) && (2 == result.size))
    result
  end

  def self.get_language_list_string()
    LANGUAGE_INFO.map { |_, info| "#{info[1]} (#{info[0]})" }.sort.join(", ")
  end

  def self.fill_default_guess_command(commands, translation_map)
    # Generate .t command to guess-translate with the default service.
    service = DEFAULT_SERVICE
    service_record = KNOWN_SERVICES[service]
    prefix = service_record[:prefix]

    cmd = "#{prefix}t".to_sym<commit_msg>Add Dutch and Swahili to Google translate command<commit_after># encoding: utf-8
# This file is part of the K5 bot project.
# See files README.md and COPYING for copyright and licensing information.

# Translate plugin

require_relative '../../IRCPlugin'
require 'rubygems'
require 'nokogiri'
require 'net/http'
require 'json'

class Translate < IRCPlugin
  Description = "Uses translation engines to translate between languages."
  Dependencies = [ :Language ]

  def self.to_lang_key(x)
    x.to_sym
  end

  def self.make_lang_service_format_map(verbatim_array, modifications_hash = {})
    result = {}
    verbatim_array.each do |x|
      result[to_lang_key(x)] = x
    end
    modifications_hash.each do |k, v|
      result[to_lang_key(k)] = v
    end
    result
  end

  def self.lang_to_service_format(l_from, l_to, possibles)
    return nil unless possibles.include? l_from # "Can't translate from #{l_from} with #{service}"
    return nil unless ((possibles.include? l_to) && !(:auto.eql? l_to)) # "Can't translate to #{l_to} with #{service}"
    [possibles[l_from], possibles[l_to]]
  end

  GOOGLE_SUPPORTED = make_lang_service_format_map(%w(auto en ja ko fr pt de it es no ru fi hu sv da pl lt nl sw), {:zh => 'zh-CN', :tw => 'zh-TW'})
  HONYAKU_SUPPORTED = make_lang_service_format_map(%w(en ja ko fr pt zh de it es))
  EXCITE_SUPPORTED = make_lang_service_format_map([], {:en => 'EN', :ja => 'JA'})
  KNOWN_SERVICES = {
      :Google => {:prefix=>'g', :languages=>GOOGLE_SUPPORTED, :translator=>:google_translate},
      :Honyaku => {:prefix=>'h', :languages=>HONYAKU_SUPPORTED, :translator=>:honyaku_translate},
      :Excite => {:prefix=>'x', :languages=>EXCITE_SUPPORTED, :translator=>:excite_translate},
  }
  DEFAULT_SERVICE = :Honyaku
  DEFAULT_SERVICE_LANGUAGES = {:en=>'e', :ja=>'j', :ko=>'k', :zh=>'c'}

  # Internal unified language id =>
  # [Shortcut form for commands, Language description for help]
  LANGUAGE_INFO = {
      :auto => ['_', 'Auto-detected language'],
      :en => ['en', 'English'],
      :ja => ['ja', 'Japanese'],
      :zh => ['zh', 'Simplified Chinese'],
      :tw => ['tw', 'Traditional Chinese'],
      :ko => ['ko', 'Korean'],
      :fr => ['fr', 'French'],
      :pt => ['pt', 'Portuguese'],
      :de => ['de', 'German'],
      :it => ['it', 'Italian'],
      :es => ['es', 'Spanish'],
      :no => ['no', 'Norwegian'],
      :ru => ['ru', 'Russian'],
      :fi => ['fi', 'Finnish'],
      :hu => ['hu', 'Hungarian'],
      :sv => ['sv', 'Swedish'],
      :da => ['da', 'Danish'],
      :pl => ['pl', 'Polish'],
      :lt => ['lt', 'Lithuanian'],
      :nl => ['nl', 'Dutch'],
      :sw => ['sw', 'Swahili'],
  }

  def self.get_language_info(lang)
    result = LANGUAGE_INFO[lang]
    raise "Cannot find language info for #{lang}" unless (result && (result.instance_of? Array) && (2 == result.size))
    result
  end

  def self.get_language_list_string()
    LANGUAGE_INFO.map { |_, info| "#{info[1]} (#{info[0]})" }.sort.join(", ")
  end

  def self.fill_default_guess_command(commands, translation_map)
    # Generate .t command to guess-translate with the default service.
    service = DEFAULT_SERVICE
    service_record = KNOWN_SERVICES[service]
    prefix = service_record[:prefix]

    cmd = "#{prefix}t".to_sym
{'fraction_non_alphanumeric': 0.17541136293076684, 'fraction_numerical': 0.001552312946289972, 'mean_word_length': 3.7514749262536875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '908962', 'n_tokens_mistral': 2311, 'n_tokens_neox': 2224, 'n_words': 572}