<commit_before>require_relative 'simple'
module Selector
  #
  # Feature Selection for Text Classification - HP Labs
  # http://www.google.com/patents/US20040059697
  #
  class Forman < Selector::Simple

    def label
      "forman"
    end

    #
    # generates a list of feature vetors and their labels from preprocessed data
    # @param  data_set [Array<PreprocessedData>] list of preprocessed data
    # @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
    # @param  dictionary_size [Integer] Size of a dictionary to create if non exists
    #
    # @return [Array<FeatureVector>] list of feature vectors and labels
    def generate_vectors data_set, dictionary_size=DEFAULT_DICTIONARY_SIZE
      words_and_label_per_data = extract_words data_set, true
      generate_global_dictionary words_and_label_per_data, dictionary_size

      words_per_data = words_and_label_per_data.map(&:features)
      p_map_with_index(words_per_data) do |words,index|
        word_set = words.uniq
        make_vector word_set, data_set[index]
      end
    end

    #
    # generates a list of words used as dictionary
    # @param  all_words (see #extract_words)
    # @param  size dictionary size
    #
    # @return [Array<String>] list of words
    def generate_global_dictionary all_words, size=DEFAULT_DICTIONARY_SIZE
      return unless global_dictionary.empty?

      label_counts = [0,0]
      features = all_words.reduce(Hash.new { |h, k| h[k] = [0,0] }) do |accumulator, bag|
        label = bag.label ? 1 : 0
        label_counts[label] += 1
        # only count a feature once per bag
        bag.features.uniq.each do |word|
          unless accumulator.has_key?(word)
            accumulator[word] = [0,0]
          end
          accumulator[word][label] += 1
        end
        accumulator
      end
      words = p_map(features) do |word, counts|
                next if counts.any? { |e| e==0 } # skip words only appearing in one class
                false_prositive_rate = counts[0].quo(label_counts[0])
                true_prositive_rate = counts[1].quo(label_counts[1])
                bns = cdf_inverse(true_prositive_rate) - cdf_inverse(false_prositive_rate)
                [word, bns.abs]
              end
      @global_dictionary = words.compact
                                .sort_by{|e| e[1]}
                                .last(size)
                                .map{|e| e[0] }
    end

    #
    # extracts the words of all provided data entries
    # @param  data_set [Array<PreprocessedData>] list of preprocessed data
    # @param  keep_label
    #
    # @return [Array<OpenStruct<Array<String>,Boolean>>] list of words per data entry
    def extract_words data_set, keep_label=false
      data_set.map do |data|
        extract_words_from_data data, keep_label
      end
    end

    #
    # fetches all words and two word phrases from one data entry, removes stopwords and very short words
    # @param  data [PreprocessedData] preprocessed data entry
    # @param  keep_label
    #
    # @return [OpenStruct<Array<String>,Boolean>] list of words
    def extract_words_from_data data, keep_label=false
      words = (data.data.flat_map(&:split) - stopwords).delete_if { |e| e.size <= 2 }
      features = words + words.each_cons(2).map{|e| e.join " " }
      return features unless keep_label
      OpenStruct.new(
        features: features,
        label: data.label
      )
    end

<commit_msg>add word selection setting to forman<commit_after>require_relative 'simple'
module Selector
  #
  # Feature Selection for Text Classification - HP Labs
  # http://www.google.com/patents/US20040059697
  #
  class Forman < Selector::Simple
    attr_reader :word_selection

    def initialize classification, args={}
      super
      @word_selection = args.fetch(:word_selection){ :grams1_2 }
    end

    def label
      "forman"
    end

    #
    # generates a list of feature vetors and their labels from preprocessed data
    # @param  data_set [Array<PreprocessedData>] list of preprocessed data
    # @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
    # @param  dictionary_size [Integer] Size of a dictionary to create if non exists
    #
    # @return [Array<FeatureVector>] list of feature vectors and labels
    def generate_vectors data_set, dictionary_size=DEFAULT_DICTIONARY_SIZE
      words_and_label_per_data = extract_words data_set, true
      generate_global_dictionary words_and_label_per_data, dictionary_size

      words_per_data = words_and_label_per_data.map(&:features)
      p_map_with_index(words_per_data) do |words,index|
        word_set = words.uniq
        make_vector word_set, data_set[index]
      end
    end

    #
    # generates a list of words used as dictionary
    # @param  all_words (see #extract_words)
    # @param  size dictionary size
    #
    # @return [Array<String>] list of words
    def generate_global_dictionary all_words, size=DEFAULT_DICTIONARY_SIZE
      return unless global_dictionary.empty?

      label_counts = [0,0]
      features = all_words.reduce(Hash.new { |h, k| h[k] = [0,0] }) do |accumulator, bag|
        label = bag.label ? 1 : 0
        label_counts[label] += 1
        # only count a feature once per bag
        bag.features.uniq.each do |word|
          unless accumulator.has_key?(word)
            accumulator[word] = [0,0]
          end
          accumulator[word][label] += 1
        end
        accumulator
      end
      words = p_map(features) do |word, counts|
                next if counts.any? { |e| e==0 } # skip words only appearing in one class
                false_prositive_rate = counts[0].quo(label_counts[0])
                true_prositive_rate = counts[1].quo(label_counts[1])
                bns = cdf_inverse(true_prositive_rate) - cdf_inverse(false_prositive_rate)
                [word, bns.abs]
              end
      @global_dictionary = words.compact
                                .sort_by{|e| e[1]}
                                .last(size)
                                .map{|e| e[0] }
    end

    #
    # extracts the words of all provided data entries
    # @param  data_set [Array<PreprocessedData>] list of preprocessed data
    # @param  keep_label
    #
    # @return [Array<OpenStruct<Array<String>,Boolean>>] list of words per data entry
    def extract_words data_set, keep_label=false
      data_set.map do |data|
        extract_words_from_data data, keep_label
      end
    end

    #
    # fetches all words and two word phrases from one data entry, removes stopwords and very short words
    # @param  data [PreprocessedData] preprocessed data entry
    # @param  keep_label
    #
    # @return [OpenStruct<Array<String>,Boolean>] list of words
    def extract_words_from_data data, keep_label=false
      words = (data.data.flat_map(&:split) - stopwords).delete_if { |e| e.size <= 2 }
      features =  case word_selection
                  when :grams1_2
                    words + words.each_cons(2).map{|e| e.join " " }
                  when :grams1_2_3
                    words +
                      words.each_cons(2).map{|e| e.join " " } +
                      words.each_cons(3).map{|e| e.join " " }
                  when :grams1_2_3_4
                    words +
                      words.each_cons(2).map{|e| e.join " " } +
                      words.each_cons(3).map{|e| e.join " " } +
                      words.each_cons(4).map{|e| e.join " " }
                  end
      return features unless keep_label
      OpenStruct.new(
        features: features,
        label: data.label
      )
    end


{'fraction_non_alphanumeric': 0.0880198796756474, 'fraction_numerical': 0.009939837823698667, 'mean_word_length': 2.3075259515570936, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5812033', 'n_tokens_mistral': 2401, 'n_tokens_neox': 2265, 'n_words': 595}