<commit_before>require 'matrix'

module ML
  module Data
    # Generating sample points on 2D plane
    class Generator2D
      # Generate point from line
      #
      # @param [Array] coef [a,b,c] for ax+by+c=0
      # @param [Number] x x value
      # @return [Array] point
      def self.point_from_line coef, x
        [x, (-coef[2]-(coef[0] * x))/coef[1]]
      end

      # Initialize a generator
      #
      # @param [Integer] x_range x range
      # @param [Integer] y_range y range
      def initialize x_range = 100, y_range = 100
        @x_range = x_range
        @y_range = y_range
      end

      # Generate two groups of points on 2d plain
      #
      # @param [Integer] points the number of points of each set
      # @param [Array] coef [a,b,c] for ax+by+c=0
      # @return [Hash] key: points, value: supervised value
      def points_2d points, coef = [-1.0, 1.0, 0.0]
        result = {}
        # for each group
        [1, -1].each do |grp|
          points.times do
            while true
              point = generate_point
              prod = Matrix.column_vector(point).transpose * Matrix.column_vector(coef)
              if (prod[0,0] <=> 0) == grp
                result[point] = grp
                break
              end
            end
          end
        end
        result
      end

    private
      def generate_point
        [@x_range * rand, @y_range * rand, 1.0]
      end
    end

    # General generator for n-dimentional space
    class Generator
      # Initial generator
      #
      # @param [Integer] dim dimension
      def initialize dim
        @dim = dim
      end

      # Generate two groups of points
      #
      # @param [Integer] points the number of points of each set
      # @param [Array] coef array of the size of dimension to specify the hyper plane
      # @return [Hash] key: points, value: supervised value
      def points points, coef
        result = {}
        # for each group
        [1, -1].each do |grp|
          points.times do
            while true
              point = Generator.generate_vector(@dim, 100)
              prod = Matrix.column_vector(point).transpose * Matrix.column_vector(coef)
              if (prod[0,0] <=> 0) == grp
                result[point] = grp
                break
              end
            end
          end
        end
        result
      end

      # Generating a random vector
      #
      # @param [Integer] dim the dimension of the vector
      # @param [Integer] scale the scale of each component
      # @return [Array] random vector
      def self.generate_vector dim, scale = 1
        result = Array.new(dim) { (rand - 0.5) * scale } <commit_msg>Adding noise generation<commit_after>require 'matrix'

module ML
  module Data
    # Generating sample points on 2D plane
    class Generator2D
      # Generate point from line
      #
      # @param [Array] coef [a,b,c] for ax+by+c=0
      # @param [Number] x x value
      # @return [Array] point
      def self.point_from_line coef, x
        [x, (-coef[2]-(coef[0] * x))/coef[1]]
      end

      # Initialize a generator
      #
      # @param [Integer] x_range x range
      # @param [Integer] y_range y range
      # @param [Numeric] noise the percentage of noise
      def initialize x_range = 100, y_range = 100, noise = 0
        @x_range = x_range
        @y_range = y_range
        @noise = noise
      end

      # Generate two groups of points on 2d plain
      #
      # @param [Integer] points the number of points of each set
      # @param [Array] coef [a,b,c] for ax+by+c=0
      # @return [Hash] key: points, value: supervised value
      def points_2d points, coef = [-1.0, 1.0, 0.0]
        result = {}
        # for each group
        [1, -1].each do |grp|
          points.times do
            while true
              point = generate_point
              prod = Matrix.column_vector(point).transpose * Matrix.column_vector(coef)
              if (prod[0,0] <=> 0) == grp
                result[point] = grp
                break
              end
            end
          end
        end

        if @noise > 0
          flipping = (points * @noise).to_i
          order = (1...points).to_a.shuffle
          for i in 0...flipping
            points[points.keys[order[i]]] *= -1
          end
        end

        result
      end

    private
      def generate_point
        [@x_range * rand, @y_range * rand, 1.0]
      end
    end

    # General generator for n-dimentional space
    class Generator
      # Initial generator
      #
      # @param [Integer] dim dimension
      # @param [Numeric] noise the percentage of noise
      def initialize dim, noise = 0
        @dim = dim
        @noise = noise
      end

      # Generate two groups of points
      #
      # @param [Integer] points the number of points of each set
      # @param [Array] coef array of the size of dimension to specify the hyper plane
      # @return [Hash] key: points, value: supervised value
      def points points, coef
        result = {}
        # for each group
        [1, -1].each do |grp|
          points.times do
            while true
              point = Generator.generate_vector(@dim, 100)
              prod = Matrix.column_vector(point).transpose * Matrix.column_vector(coef)
              if (prod[0,0] <=> 0) == grp
                result[point] = grp
                break
              end
            end
          end
        end

        if @noise > 0
          flipping = (points * @noise).to_i
          order = (1...points).to_a.shuffle
          for i in 0...flipping
            points[points.keys[order[i]]] *= -1
          end
        end

        result
      end

      # Generating a random vector
      #
      # @param [Integer] dim the dimension of the vector
      # @param [Integer] scale the scale of each component
      # @return [Array] random vector
      def self.generate_vector dim, scale = 1
        result = Array.new(dim) { (rand - 0.5) * scale } 
{'fraction_non_alphanumeric': 0.09142475809142475, 'fraction_numerical': 0.014681348014681348, 'mean_word_length': 1.603126356925749, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179619', 'n_tokens_mistral': 1819, 'n_tokens_neox': 1743, 'n_words': 588}