<commit_before>module Libsvmffi
  #
  #
  #
  class Model
 
    attr_accessor :parameters, :problem, :svm_model
    attr_accessor :examples
    attr_accessor :labels, :features 
    attr_accessor :elements

    def initialize(options = {})

      @parameters = Parameters.new      
      
      # TODO merge defaults options hash.
      @parameters[:svm_type] = C_SVC;
      @parameters[:kernel_type] = RBF;
      @parameters[:degree] = 3;
      @parameters[:gamma] = 0;  # 1/num_features
      @parameters[:coef0] = 0;
      @parameters[:nu] = 0.5;
      @parameters[:cache_size] = 100;
      @parameters[:C] = 1;
      @parameters[:eps] = 0.001;
      @parameters[:p] = 0.1;
      @parameters[:shrinking] = 1;
      @parameters[:probability] = 0;
      @parameters[:nr_weight] = 0;
      @parameters[:weight_label] = nil;
      @parameters[:weight] = nil;
      
      @elements = 0
      @examples, @labels, @features = [], [], []
    end

    #
    # Adds an example which will be used upon trainining
    #
    def add(label, features)
      @labels.push label unless @labels.include? label
      indexed_features = {}
      features.each do |k, v|
        @features.push k unless @features.includes? k
        indexed_features[@features.index(k)] = v
      end

      @examples.push({@labels.index(label) => indexed_features})

      @elements += features.length + 1 # also -1 terminator, see libsvm readme.
    end

    #
    # Build libsvm model
    # 
    def train

      @problem = Problem.new
      @problem.l = @examples.length
      @problem.y = @examples.keys
      @problem.x = FFI::MemoryPointer.new(:pointer, @problem.l)
      @x_space = FFI::MemoryPointer.new(:pointer, @elements)

      space_index = 0
      @examples.values.each_with_index do |e, i|
        @problem.x[i] = @x_space[space_index].pointer
      
        e.merge!({-1 => 0}) #terminator
        e.each do |k, v|
          n = Node.new
          n[:index] = k
          n[:value] = v
          @x_space[space_index] = n.pointer
          space_index += 1
        end
      end
      
      # @svm_model = svm_train @problem.pointer, @paramaters.pointer
      
    end

    #
    # Save to file
    #
    def save(filename = "thing")
       #svm_save_model filename, @svm_model.pointer
    end
    
   
  end
end
<commit_msg>another one<commit_after>module Libsvmffi
  #
  #
  #
  class Model
 
    attr_accessor :parameters, :problem, :svm_model
    attr_accessor :examples
    attr_accessor :labels, :features 
    attr_accessor :elements

    def initialize(options = {})

      @parameters = Parameters.new      
      
      # TODO merge defaults options hash.
      @parameters[:svm_type] = C_SVC;
      @parameters[:kernel_type] = RBF;
      @parameters[:degree] = 3;
      @parameters[:gamma] = 0;  # 1/num_features
      @parameters[:coef0] = 0;
      @parameters[:nu] = 0.5;
      @parameters[:cache_size] = 100;
      @parameters[:C] = 1;
      @parameters[:eps] = 0.001;
      @parameters[:p] = 0.1;
      @parameters[:shrinking] = 1;
      @parameters[:probability] = 0;
      @parameters[:nr_weight] = 0;
      @parameters[:weight_label] = nil;
      @parameters[:weight] = nil;
      
      @elements = 0
      @examples, @labels, @features = [], [], []
    end

    #
    # Adds an example which will be used upon trainining
    #
    def add(label, features)
      @labels.push label unless @labels.include? label
      indexed_features = {}
      features.each do |k, v|
        @features.push k unless @features.include? k
        indexed_features[@features.index(k)] = v
      end

      @examples.push({@labels.index(label) => indexed_features})

      @elements += features.length + 1 # also -1 terminator, see libsvm readme.
    end

    #
    # Build libsvm model
    # 
    def train

      @problem = Problem.new
      @problem.l = @examples.length
      @problem.y = @examples.keys
      @problem.x = FFI::MemoryPointer.new(:pointer, @problem.l)
      @x_space = FFI::MemoryPointer.new(:pointer, @elements)

      space_index = 0
      @examples.values.each_with_index do |e, i|
        @problem.x[i] = @x_space[space_index].pointer
      
        e.merge!({-1 => 0}) #terminator
        e.each do |k, v|
          n = Node.new
          n[:index] = k
          n[:value] = v
          @x_space[space_index] = n.pointer
          space_index += 1
        end
      end
      
      # @svm_model = svm_train @problem.pointer, @paramaters.pointer
      
    end

    #
    # Save to file
    #
    def save(filename = "thing")
       #svm_save_model filename, @svm_model.pointer
    end
    
   
  end
end

{'fraction_non_alphanumeric': 0.12697724810400868, 'fraction_numerical': 0.011700975081256771, 'mean_word_length': 2.1616438356164385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422046', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1489, 'n_words': 329}