<commit_before>require "csv"
require "nkf"
require 'uri'

class Prefix < ActiveRecord::Base

  belongs_to :endpoint
  validates :allowed_uri, format: URI::regexp(%w(http https ftp)), allow_blank: true
  validates :denied_uri, format: URI::regexp(%w(http https ftp)), allow_blank: true

  class FormatError < StandardError
  end

  def self.import_csv(params)
    prefixes = CSV.parse(params[:endpoint][:file].read, {headers: true}).map do |row|
      prefix = Prefix.new
      prefix.endpoint_id = params[:id]
      prefix.allowed_uri = NKF::nkf("-w", row[0].to_s)
      prefix.denied_uri = NKF::nkf("-w", row[1].to_s)
      case_sensitive = NKF::nkf("-w", row[2].to_s)
      if case_sensitive.present?
        if %w(true false).include?(case_sensitive)
          prefix.case_sensitive = case_sensitive
        else
          raise FormatError.new("Failed to import: '#{case_sensitive}' is invalid as case sensitiveness. It must be true or false")
        end
      end

      unless prefix.valid?
        raise FormatError.new("Failed to import: #{prefix.errors.full_messages} URI must start with 'http://', 'https://' or 'ftp://'.")
      end
      prefix
    end
    prefixes.each{ |prefix| prefix.save }
    nil
  end

  def self.validates_params(params)
    if params[:endpoint].nil?
      return 'CSV file is required'
    end
    nil
  end

end<commit_msg>add validation of uri presence<commit_after>require "csv"
require "nkf"
require 'uri'

class Prefix < ActiveRecord::Base

  belongs_to :endpoint
  validates :allowed_uri, format: URI::regexp(%w(http https ftp)), allow_blank: true
  validates :denied_uri, format: URI::regexp(%w(http https ftp)), allow_blank: true
  validate :uri_present

  class FormatError < StandardError
  end

  def self.import_csv(params)
    prefixes = CSV.parse(params[:endpoint][:file].read, {headers: true}).map do |row|
      prefix = Prefix.new
      prefix.endpoint_id = params[:id]
      prefix.allowed_uri = NKF::nkf("-w", row[0].to_s)
      prefix.denied_uri = NKF::nkf("-w", row[1].to_s)
      case_sensitive = NKF::nkf("-w", row[2].to_s)
      if case_sensitive.present?
        if %w(true false).include?(case_sensitive)
          prefix.case_sensitive = case_sensitive
        else
          raise FormatError.new("Failed to import: '#{case_sensitive}' is invalid as case sensitiveness. It must be true or false")
        end
      end

      unless prefix.valid?
        raise FormatError.new("Failed to import: #{prefix.errors.full_messages} URI must start with 'http://', 'https://' or 'ftp://'.")
      end
      prefix
    end
    prefixes.each{ |prefix| prefix.save }
    nil
  end

  def self.validates_params(params)
    if params[:endpoint].nil?
      return 'CSV file is required'
    end
    nil
  end


  private

  def uri_present
    unless allowed_uri.present? || denied_uri.present?
      errors.add(:base, "At least one of allowed_uri and denied_uri must be present.")
    end
  end

end
{'fraction_non_alphanumeric': 0.12220717670954638, 'fraction_numerical': 0.002031144211238998, 'mean_word_length': 3.1502808988764044, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3362196', 'n_tokens_mistral': 976, 'n_tokens_neox': 940, 'n_words': 252}