<commit_before>module Linkscape
  class Request
    require 'net/http'
    require 'uri'
    require 'cgi'
    # require 'base64'
    # require 'rubygems'
    # require 'hmac-sha1'
    
    attr_accessor :requestURL
    
    URL_TEMPLATE = %Q[http://:apiHost:/:apiRoot:/:api:/:url:?AccessID=:accessID:&Expires=:expiration:&Signature=:signature:]

    def self.run(options)
      self.new(options).run
    end
    
    def initialize(options)
     
      new_vals = {}
      if options[:url] 
        case options[:url]
        when String
          new_vals = {:url => CGI::escape(options[:url].sub(/^https?:\/\//, '')) }
        when Array
          @body = options[:url].collect{ |u| u.sub(/^https?:\/\//, '') }
          new_vals = {:url => ""}
        else
          raise "URL most be a String or an Array"
        end
      end
     <commit_msg>Add a check to URL length for the request to verify it is not too long<commit_after>module Linkscape
  class Request
    require 'net/http'
    require 'uri'
    require 'cgi'
    # require 'base64'
    # require 'rubygems'
    # require 'hmac-sha1'
    
    attr_accessor :requestURL
    
    URL_TEMPLATE = %Q[http://:apiHost:/:apiRoot:/:api:/:url:?AccessID=:accessID:&Expires=:expiration:&Signature=:signature:]
    MAX_URL_LENGTH = 500

    def self.run(options)
      self.new(options).run
    end
    
    def initialize(options)
     
      new_vals = {}
      if options[:url] 
        case options[:url]
        when String
          new_vals = {:url => CGI::escape(options[:url].sub(/^https?:\/\//, '')) }
        when Array
          @body = options[:url].collect{ |u| u.sub(/^https?:\/\//, '') }
          new_vals = {:url => ""}
        else
          raise "URL most be a String or an Array"
        end
      end

      if Array(new_vals[:url]).any? { |u| u.length > MAX_URL_LENGTH }
        raise ArgumentError.new("Request URLs must be < #{MAX_URL_LENGTH} long")
      end
     
{'fraction_non_alphanumeric': 0.1474820143884892, 'fraction_numerical': 0.0046248715313463515, 'mean_word_length': 2.1710097719869705, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7286164', 'n_tokens_mistral': 652, 'n_tokens_neox': 630, 'n_words': 142}