<commit_before>#--
###############################################################################
#                                                                             #
# lsi4r -- Latent semantic indexing for Ruby                                  #
#                                                                             #
# Copyright (C) 2014 <NAME>                                               #
#                                                                             #
# Authors:                                                                    #
#     <NAME> <<EMAIL>>                                       #
#                                                                             #
# lsi4r is free software; you can redistribute it and/or modify it under the  #
# terms of the GNU Affero General Public License as published by the Free     #
# Software Foundation; either version 3 of the License, or (at your option)   #
# any later version.                                                          #
#                                                                             #
# lsi4r is distributed in the hope that it will be useful, but WITHOUT ANY    #
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
# more details.                                                               #
#                                                                             #
# You should have received a copy of the GNU Affero General Public License    #
# along with lsi4r. If not, see <http://www.gnu.org/licenses/>.               #
#                                                                             #
###############################################################################
#++

require 'forwardable'
require 'gsl'

class Lsi4R

  include Enumerable

  extend Forwardable

  DEFAULT_EPSILON   = Float::EPSILON * 10

  DEFAULT_TRANSFORM = :tfidf

  DEFAULT_CUTOFF    = 0.75

  class << self

    def build(items, options = {})
      lsi = new(items)
      lsi if lsi.build(options)
    end

    def each_norm(items, options = {}, build_options = {}, &block)
      lsi = new(items)
      lsi.each_norm(nil, options, &block) if lsi.build(build_options)
    end

  end

  def initialize(items = {})
    reset
    items.each { |k, v| self[k] = v || k }
  end

  def_delegators :@hash, :[], :each, :include?, :key, :keys, :size

  def_delegator  :@hash, :values,    :docs
  def_delegator  :@hash, :values_at, :docs_at

  def_delegator  :@list, :keys, :terms

  alias_method :doc, :[]

  def []=(key, value)
    @hash[key] = Doc.new(key, value, @list, @freq)
  end

  def add(key, value = key)
    self[key] = value
    self
  end

  def <<(value)
    add(value.object_id, value)
  end

  def each_vector(key = nil, norm = true)
    return enum_for(:each_vector, key, norm) unless block_given?

    (key ? [self[key]] : docs).each { |doc|
      if doc && vec = norm ? doc.norm : doc.vector
        yield doc, vec
      end
    }
  end

  # min:: minimum value to consider
  # abs:: minimum absolute value to consider
  # nul:: exclude null values (true or Float)
  # new:: exclude original terms / only yield new ones
  def each_term(key = nil, options = {})
    return enum_for(:each_term, key, options) unless block_given?

    min, abs, nul, new = options.values_at(:min, :abs, :nul, :new)
    nul = DEFAULT_EPSILON if nul == true

    list = @invlist

    each_vector(key, options[:norm]) { |doc, vec|
      vec.enum_for(:each).with_index { |v, i|
        yield doc, list[i], v unless v.nan? ||
                                     (min && v < min) ||
                                     (abs && v.abs < abs) ||
                                     (nul && v.abs < nul) ||
                                     (new && doc.include?(i))
      }
    }
  end

  def each_norm(key = nil, options = {}, &block)
    each_term(key, options.merge(norm: true), &block)
  end

  def related(key, num = 5)
    if doc = self[key] and norm = doc.norm
      temp = sort_by { |k, v| -norm * v.norm.col }
      temp.map! { |k,| k }.delete(key)
      temp[0, num]
    end
  end

  def related_score(key, num = 5)
    if doc = self[key] and norm = doc.norm
      temp = map { |k, v| [k, norm * v.norm.col] }.sort_by { |_, i| -i }
      temp.delete(temp.assoc(key))
      temp[0, num]
    end
  end

  def build(options = {})
    build!(docs, @list, options.is_a?(Hash) ?
      options : { cutoff: options }) if size > 1
  end

  def reset
    @hash, @list, @freq, @invlist =
      {}, Hash.new { |h, k| h[k] = h.size }, Hash.new(0), {}
  end

  def inspect
    '%s@%d/%d' % [self.class, size, @list.size]
  end<commit_msg>lib/lsi4r.rb (related{,_score}): Refactored.<commit_after>#--
###############################################################################
#                                                                             #
# lsi4r -- Latent semantic indexing for Ruby                                  #
#                                                                             #
# Copyright (C) 2014 <NAME>                                               #
#                                                                             #
# Authors:                                                                    #
#     <NAME> <<EMAIL>>                                       #
#                                                                             #
# lsi4r is free software; you can redistribute it and/or modify it under the  #
# terms of the GNU Affero General Public License as published by the Free     #
# Software Foundation; either version 3 of the License, or (at your option)   #
# any later version.                                                          #
#                                                                             #
# lsi4r is distributed in the hope that it will be useful, but WITHOUT ANY    #
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
# more details.                                                               #
#                                                                             #
# You should have received a copy of the GNU Affero General Public License    #
# along with lsi4r. If not, see <http://www.gnu.org/licenses/>.               #
#                                                                             #
###############################################################################
#++

require 'forwardable'
require 'gsl'

class Lsi4R

  include Enumerable

  extend Forwardable

  DEFAULT_EPSILON   = Float::EPSILON * 10

  DEFAULT_TRANSFORM = :tfidf

  DEFAULT_CUTOFF    = 0.75

  class << self

    def build(items, options = {})
      lsi = new(items)
      lsi if lsi.build(options)
    end

    def each_norm(items, options = {}, build_options = {}, &block)
      lsi = new(items)
      lsi.each_norm(nil, options, &block) if lsi.build(build_options)
    end

  end

  def initialize(items = {})
    reset
    items.each { |k, v| self[k] = v || k }
  end

  def_delegators :@hash, :[], :each, :include?, :key, :keys, :size

  def_delegator  :@hash, :values,    :docs
  def_delegator  :@hash, :values_at, :docs_at

  def_delegator  :@list, :keys, :terms

  alias_method :doc, :[]

  def []=(key, value)
    @hash[key] = Doc.new(key, value, @list, @freq)
  end

  def add(key, value = key)
    self[key] = value
    self
  end

  def <<(value)
    add(value.object_id, value)
  end

  def each_vector(key = nil, norm = true)
    return enum_for(:each_vector, key, norm) unless block_given?

    (key ? [self[key]] : docs).each { |doc|
      if doc && vec = norm ? doc.norm : doc.vector
        yield doc, vec
      end
    }
  end

  # min:: minimum value to consider
  # abs:: minimum absolute value to consider
  # nul:: exclude null values (true or Float)
  # new:: exclude original terms / only yield new ones
  def each_term(key = nil, options = {})
    return enum_for(:each_term, key, options) unless block_given?

    min, abs, nul, new = options.values_at(:min, :abs, :nul, :new)
    nul = DEFAULT_EPSILON if nul == true

    list = @invlist

    each_vector(key, options[:norm]) { |doc, vec|
      vec.enum_for(:each).with_index { |v, i|
        yield doc, list[i], v unless v.nan? ||
                                     (min && v < min) ||
                                     (abs && v.abs < abs) ||
                                     (nul && v.abs < nul) ||
                                     (new && doc.include?(i))
      }
    }
  end

  def each_norm(key = nil, options = {}, &block)
    each_term(key, options.merge(norm: true), &block)
  end

  def related(key, num = 5)
    each_vector(key) { |_, vec|
      tmp, del = block_given? ? yield(vec) :
        [sort_by { |_, v| -vec * v.norm.col }.map! { |k,| k }]

      tmp.delete(del || key)

      return tmp[0, num]
    }

    nil
  end

  def related_score(key, num = 5, threshold = 0)
    related(key, num) { |vec|
      [tmp = map { |k, v|
        score = vec * v.norm.col
        [k, score] if score > threshold
      }.compact.sort_by { |_, i| -i }, tmp.assoc(key)]
    }
  end

  def build(options = {})
    build!(docs, @list, options.is_a?(Hash) ?
      options : { cutoff: options }) if size > 1
  end

  def reset
    @hash, @list, @freq, @invlist =
      {}, Hash.new { |h, k| h[k] = h.size }, Hash.new(0), {}
  end

  def inspect
    '%s@%d/%d' % [self.class, size, @list.size]
  end
{'fraction_non_alphanumeric': 0.14244216205000518, 'fraction_numerical': 0.004461043676729951, 'mean_word_length': 1.438037430450177, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 76, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2166162', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2662, 'n_words': 804}