<commit_before>require 'oily_png'
require 'base64'

module Indico
  def self.preprocess(image, size, batch)
    # image is [[f,f,f,f, ...], ..] or [[[f,f,f],[f,f,f],[f,f,f]], ...]
    if batch
      # Batch Request
      im_array = Array.new

      # process each image
      image.each do |_image|
        im_array.push(preprocess(_image, size, false))
      end

      return im_array
    end

    if image.class == String
      decoded_image = handle_string_input(image)
    elsif image.class == Array
      decoded_image = handle_array_input(image)
    else
      raise Exception.new("Image input must be nested array of pixels, filename, or base64 string")
    end

    # Resize and export base64 encoded string
    return decoded_image.resize(size, size).to_data_url.gsub("data:image/png;base64," ,"")
  end

  def self.handle_string_input(str)
    # Handles string input
    if File.file?(str)
      # Handling File Inputs
      image = load_image(str)
      return handle_array_input(image)
    end

    regex = %r{^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$}.match(str)
    # Base 64 handling
    if regex == nil
      raise Exception.new("String is not a valid absolute filepath or base64 string")
    else
      return ChunkyPNG::Image.from_data_url("data:image/png;base64," + str.gsub("data:image/png;base64," ,""))
    end
  end

  def self.handle_array_input(image)
    # Handles properly formatting and loading array of pixels
    # Single Request
    dimens = get_dimension(image)
    isFloat = array_contains_float(image, dimens)

    if dimens.size > 3
      raise Exception.new("Nested array of image must be [[p, p, ...]] or [[[r,g,b], [r,g,b], ...]]")
    end

    decoded_image = ChunkyPNG::Image.new(dimens[0], dimens[1])

    # Manually loading the pixels into the ChunkyImage object
    (0..dimens[0] - 1).each do |x|
      (0..dimens[1] - 1).each do |y|
        pixel = image[x][y]
        if pixel.is_a?(Array)
          if isFloat
            pixel.map! {|a| (255 * a).to_i}
          end
          decoded_image.set_pixel(x,y, ChunkyPNG::Color.rgb(pixel[0], pixel[1], pixel[2]))
        else
          if isFloat
            pixel = (pixel * 255).to_i
          end
          decoded_image.set_pixel(x,y, ChunkyPNG::Color.rgb(pixel, pixel, pixel))
        end
      end
    end

    return decoded_image
  end


  def self.get_dimension(array)
    return [] unless array.is_a?(Array)
    return [array.size] + get_dimension(array[0])
  end

  def self.array_contains_float(array, dimens)
    # Determines whether or not the array contains floats
    if not array.is_a?(Array)
      return array.class == Float
    end
    elem = array[0]
    (0..dimens.size - 2).each do |i|
      elem = elem[0]
    end

    return elem.class == Float
  end

  def self.get_rgb(value)
    # Returns Integer encoding of RGB value used by ChunkyPNG
    return [
      ChunkyPNG::Color.r(value),
      ChunkyPNG::Color.g(value),
      ChunkyPNG::Color.b(value)
    ]
  end

  def self.load_image(filepath)
    # Loads and populates properly formatted array from filepath
    image_load = ChunkyPNG::Image.from_file(filepath)
    image = Array.new(image_load.width) {Array.new(image_load.height)}
    # image still good

    (0..image_load.height - 1).each do |x|
      (0..image_load.width - 1).each do |y|
        image[x][y] = get_rgb(image_load[x, y])
      end
    end
    return image
  end
end<commit_msg>FIX: Use b64 and warn deprecation on array input<commit_after>require 'oily_png'
require 'base64'

module Indico
  def self.preprocess(image, size, batch)
    # image is [[f,f,f,f, ...], ..] or [[[f,f,f],[f,f,f],[f,f,f]], ...]
    if batch
      # Batch Request
      im_array = Array.new

      # process each image
      image.each do |_image|
        im_array.push(preprocess(_image, size, false))
      end

      return im_array
    end

    if image.class == String
      decoded_image = handle_string_input(image)
    elsif image.class == Array
      warn "Warning! Array input as image will be deprecated in the next major release.\n Consider using filepaths or base64 encoded strings"
      decoded_image = handle_array_input(image)
    else
      raise Exception.new("Image input must be nested array of pixels, filename, or base64 string")
    end

    # Resize and export base64 encoded string
    return decoded_image.resize(size, size).to_data_url.gsub("data:image/png;base64," ,"")
  end

  def self.handle_string_input(str)
    # Handles string input
    if File.file?(str)
      # Handling File Inputs
      return ChunkyPNG::Image.from_file(str)
    end

    regex = %r{^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$}.match(str)
    # Base 64 handling
    if regex == nil
      raise Exception.new("String is not a valid absolute filepath or base64 string")
    else
      return ChunkyPNG::Image.from_data_url("data:image/png;base64," + str.gsub("data:image/png;base64," ,""))
    end
  end

  def self.handle_array_input(image)
    # Handles properly formatting and loading array of pixels
    # Single Request
    dimens = get_dimension(image)
    isFloat = array_contains_float(image, dimens)

    if dimens.size > 3
      raise Exception.new("Nested array of image must be [[p, p, ...]] or [[[r,g,b], [r,g,b], ...]]")
    end

    decoded_image = ChunkyPNG::Image.new(dimens[0], dimens[1])

    # Manually loading the pixels into the ChunkyImage object
    (0..dimens[0] - 1).each do |x|
      (0..dimens[1] - 1).each do |y|
        pixel = image[x][y]
        if pixel.is_a?(Array)
          if isFloat
            pixel.map! {|a| (255 * a).to_i}
          end
          decoded_image.set_pixel(x,y, ChunkyPNG::Color.rgb(pixel[0], pixel[1], pixel[2]))
        else
          if isFloat
            pixel = (pixel * 255).to_i
          end
          decoded_image.set_pixel(x,y, ChunkyPNG::Color.rgb(pixel, pixel, pixel))
        end
      end
    end

    return decoded_image
  end


  def self.get_dimension(array)
    return [] unless array.is_a?(Array)
    return [array.size] + get_dimension(array[0])
  end

  def self.array_contains_float(array, dimens)
    # Determines whether or not the array contains floats
    if not array.is_a?(Array)
      return array.class == Float
    end
    elem = array[0]
    (0..dimens.size - 2).each do |i|
      elem = elem[0]
    end

    return elem.class == Float
  end

  def self.get_rgb(value)
    # Returns Integer encoding of RGB value used by ChunkyPNG
    return [
      ChunkyPNG::Color.r(value),
      ChunkyPNG::Color.g(value),
      ChunkyPNG::Color.b(value)
    ]
  end
end
{'fraction_non_alphanumeric': 0.129557095510696, 'fraction_numerical': 0.016571256402530883, 'mean_word_length': 2.6538249862410566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '69998', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2285, 'n_words': 560}