<commit_before># encoding: utf-8
# This file is part of the K5 bot project.
# See files README.md and COPYING for copyright and licensing information.

# YEDICT plugin

require 'rubygems'
require 'bundler/setup'
require 'sequel'

require_relative '../../IRCPlugin'
require_relative '../../SequelHelpers'
require_relative 'YEDICTEntry'

class YEDICT < IRCPlugin
  include SequelHelpers

  Description = 'A YEDICT plugin.'
  Commands = {
    :cn => 'looks up a Cantonese word in YEDICT',
    :cnen => 'looks up an English word in YEDICT',
  }
  Dependencies = [ :Language, :Menu ]

  def afterLoad
    load_helper_class(:YEDICTEntry)

    @l = @plugin_manager.plugins[:Language]
    @m = @plugin_manager.plugins[:Menu]

    @db = database_connect("sqlite://#{(File.dirname __FILE__)}/yedict.sqlite", :encoding => 'utf8')

    @hash_yedict = load_dict(@db)
  end

  def beforeUnload
    @m.evict_plugin_menus!(self.name)

    @hash_yedict = nil

    database_disconnect(@db)
    @db = nil

    @m = nil
    @l = nil

    unload_helper_class(:YEDICTEntry)

    nil
  end

  def on_privmsg(msg)
    case msg.bot_command
    when :cn
      word = msg.tail
      return unless word
      yedict_lookup = lookup([word], [:cantonese, :jyutping])
      reply_with_menu(msg, generate_menu(format_description_unambiguous(yedict_lookup), "\"#{word}\" in YEDICT"))
    when :cnen
      word = msg.tail
      return unless word
      yedict_lookup = keyword_lookup(split_into_keywords(word))
      reply_with_menu(msg, generate_menu(format_description_show_hanzi(yedict_lookup), "\"#{word}\" in YEDICT"))
    end
  end

  def format_description_unambiguous(lookup_result)
    amb_chk_hanzi = Hash.new(0)
    amb_chk_jyutping = Hash.new(0)

    lookup_result.each do |e|
      hanzi_list = YEDICT.format_hanzi_list(e)
      jyutping_list = YEDICT.format_jyutping_list(e)

      amb_chk_hanzi[hanzi_list] += 1
      amb_chk_jyutping[jyutping_list] += 1
    end
    render_hanzi = amb_chk_hanzi.keys.size > 1

    lookup_result.map do |e|
      hanzi_list = YEDICT.format_hanzi_list(e)

      render_jyutping = amb_chk_hanzi[hanzi_list] > 1

      [e, render_hanzi, render_jyutping]
    end
  end

  def format_description_show_hanzi(lookup_result)
    lookup_result.map do |entry|
      [entry, true, false]
    end
  end

  def self.format_hanzi_list(e)
    e.cantonese
  end

  def self.format_jyutping_list(e)
    e.jyutping
  end

  def generate_menu(lookup, name)
    menu = lookup.map do |e, render_hanzi, render_jyutping|
      hanzi_list = YEDICT.format_hanzi_list(e)
      jyutping_list = YEDICT.format_jyutping_list(e)

      description = if render_hanzi && !hanzi_list.empty? then
                      render_jyutping ? "#{hanzi_list} (#{jyutping_list})" : hanzi_list
                    elsif jyutping_list
                      jyutping_list
                    else
                      '<invalid entry>'
                    end

      MenuNodeText.new(description, e)
    end

    MenuNodeSimple.new(name, menu)
  end

  def reply_with_menu(msg, result)
    @m.put_new_menu(
      self.name,
      result,
      msg
    )
  end

  # Looks up all entries that have each given word in any
  # of the specified columns and returns the result as an array of entries
  def lookup(words, columns)
    table = @hash_yedict[:yedict_entries]

    condition = Sequel.|(*words.map do |word|
      Sequel.or(columns.map { |column| [column, word] })
    end)<commit_msg>Fix lookup() description in YEDICT<commit_after># encoding: utf-8
# This file is part of the K5 bot project.
# See files README.md and COPYING for copyright and licensing information.

# YEDICT plugin

require 'rubygems'
require 'bundler/setup'
require 'sequel'

require_relative '../../IRCPlugin'
require_relative '../../SequelHelpers'
require_relative 'YEDICTEntry'

class YEDICT < IRCPlugin
  include SequelHelpers

  Description = 'A YEDICT plugin.'
  Commands = {
    :cn => 'looks up a Cantonese word in YEDICT',
    :cnen => 'looks up an English word in YEDICT',
  }
  Dependencies = [ :Language, :Menu ]

  def afterLoad
    load_helper_class(:YEDICTEntry)

    @l = @plugin_manager.plugins[:Language]
    @m = @plugin_manager.plugins[:Menu]

    @db = database_connect("sqlite://#{(File.dirname __FILE__)}/yedict.sqlite", :encoding => 'utf8')

    @hash_yedict = load_dict(@db)
  end

  def beforeUnload
    @m.evict_plugin_menus!(self.name)

    @hash_yedict = nil

    database_disconnect(@db)
    @db = nil

    @m = nil
    @l = nil

    unload_helper_class(:YEDICTEntry)

    nil
  end

  def on_privmsg(msg)
    case msg.bot_command
    when :cn
      word = msg.tail
      return unless word
      yedict_lookup = lookup([word], [:cantonese, :jyutping])
      reply_with_menu(msg, generate_menu(format_description_unambiguous(yedict_lookup), "\"#{word}\" in YEDICT"))
    when :cnen
      word = msg.tail
      return unless word
      yedict_lookup = keyword_lookup(split_into_keywords(word))
      reply_with_menu(msg, generate_menu(format_description_show_hanzi(yedict_lookup), "\"#{word}\" in YEDICT"))
    end
  end

  def format_description_unambiguous(lookup_result)
    amb_chk_hanzi = Hash.new(0)
    amb_chk_jyutping = Hash.new(0)

    lookup_result.each do |e|
      hanzi_list = YEDICT.format_hanzi_list(e)
      jyutping_list = YEDICT.format_jyutping_list(e)

      amb_chk_hanzi[hanzi_list] += 1
      amb_chk_jyutping[jyutping_list] += 1
    end
    render_hanzi = amb_chk_hanzi.keys.size > 1

    lookup_result.map do |e|
      hanzi_list = YEDICT.format_hanzi_list(e)

      render_jyutping = amb_chk_hanzi[hanzi_list] > 1

      [e, render_hanzi, render_jyutping]
    end
  end

  def format_description_show_hanzi(lookup_result)
    lookup_result.map do |entry|
      [entry, true, false]
    end
  end

  def self.format_hanzi_list(e)
    e.cantonese
  end

  def self.format_jyutping_list(e)
    e.jyutping
  end

  def generate_menu(lookup, name)
    menu = lookup.map do |e, render_hanzi, render_jyutping|
      hanzi_list = YEDICT.format_hanzi_list(e)
      jyutping_list = YEDICT.format_jyutping_list(e)

      description = if render_hanzi && !hanzi_list.empty? then
                      render_jyutping ? "#{hanzi_list} (#{jyutping_list})" : hanzi_list
                    elsif jyutping_list
                      jyutping_list
                    else
                      '<invalid entry>'
                    end

      MenuNodeText.new(description, e)
    end

    MenuNodeSimple.new(name, menu)
  end

  def reply_with_menu(msg, result)
    @m.put_new_menu(
      self.name,
      result,
      msg
    )
  end

  # Looks up all entries that have any given word in any
  # of the specified columns and returns the result as an array of entries
  def lookup(words, columns)
    table = @hash_yedict[:yedict_entries]

    condition = Sequel.|(*words.map do |word|
      Sequel.or(columns.map { |column| [column, word] })
    end)
{'fraction_non_alphanumeric': 0.09803921568627451, 'fraction_numerical': 0.0025951557093425604, 'mean_word_length': 2.9303116147308783, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7362949', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2442, 'n_words': 531}