<commit_before>require 'active_support'
require 'active_support/builder'
require 'base64'
require 'cgi'
require 'net/https'
require 'nokogiri'
require 'nori'
require 'oauth'
require 'openssl'
require 'securerandom'
require 'uri'

module Aggcat
  class Base

    SAML_URL = 'https://oauth.intuit.com/oauth/v1/get_access_token_by_saml'

    NAMESPACE = 'http://schema.intuit.com/platform/fdatafeed/institutionlogin/v1'
    TIME_FORMAT = '%Y-%m-%dT%T.%LZ'
    DATE_FORMAT = '%Y-%m-%d'

    TIMEOUT = 120

    IGNORE_KEYS = Set.new([:'@xmlns', :'@xmlns:ns2', :'@xmlns:ns3', :'@xmlns:ns4', :'@xmlns:ns5', :'@xmlns:ns6', :'@xmlns:ns7', :'@xmlns:ns8'])

    protected

    def access_token
      token = oauth_token
      consumer = OAuth::Consumer.new(@consumer_key, @consumer_secret, {:timeout => TIMEOUT})
      OAuth::AccessToken.new(consumer, token[:key], token[:secret])
    end

    def oauth_token
      now = Time.now.utc
      if @oauth_token.nil? || @oauth_token[:expire_at] <= now
        @oauth_token = new_token(saml_message(@customer_id))
        @oauth_token[:expire_at] = now + 9 * 60 # 9 minutes
      end
      @oauth_token
    end

    def new_token(message)
      uri = URI.parse(SAML_URL)
      http = Net::HTTP.new(uri.host, uri.port)
      request = Net::HTTP::Post.new(uri.request_uri)
      request['Authorization'] = %[OAuth oauth_consumer_key="#{@consumer_key}"]
      request.set_form_data({:saml_assertion => message})
      http.use_ssl = true
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
      #http.set_debug_output($stdout)
      response = http.request(request)
      params = CGI::parse(response.body)
      {key: params['oauth_token'][0], secret: params['oauth_token_secret'][0]}
    end<commit_msg>added more fields to ignore<commit_after>require 'active_support'
require 'active_support/builder'
require 'base64'
require 'cgi'
require 'net/https'
require 'nokogiri'
require 'nori'
require 'oauth'
require 'openssl'
require 'securerandom'
require 'uri'

module Aggcat
  class Base

    SAML_URL = 'https://oauth.intuit.com/oauth/v1/get_access_token_by_saml'

    NAMESPACE = 'http://schema.intuit.com/platform/fdatafeed/institutionlogin/v1'
    TIME_FORMAT = '%Y-%m-%dT%T.%LZ'
    DATE_FORMAT = '%Y-%m-%d'

    TIMEOUT = 120

    IGNORE_KEYS = Set.new([:'@xmlns', :'@xmlns:ns2', :'@xmlns:ns3', :'@xmlns:ns4', :'@xmlns:ns5', :'@xmlns:ns6', :'@xmlns:ns7', :'@xmlns:ns8', :'@xmlns:ns9'])

    protected

    def access_token
      token = oauth_token
      consumer = OAuth::Consumer.new(@consumer_key, @consumer_secret, {:timeout => TIMEOUT})
      OAuth::AccessToken.new(consumer, token[:key], token[:secret])
    end

    def oauth_token
      now = Time.now.utc
      if @oauth_token.nil? || @oauth_token[:expire_at] <= now
        @oauth_token = new_token(saml_message(@customer_id))
        @oauth_token[:expire_at] = now + 9 * 60 # 9 minutes
      end
      @oauth_token
    end

    def new_token(message)
      uri = URI.parse(SAML_URL)
      http = Net::HTTP.new(uri.host, uri.port)
      request = Net::HTTP::Post.new(uri.request_uri)
      request['Authorization'] = %[OAuth oauth_consumer_key="#{@consumer_key}"]
      request.set_form_data({:saml_assertion => message})
      http.use_ssl = true
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
      #http.set_debug_output($stdout)
      response = http.request(request)
      params = CGI::parse(response.body)
      {key: params['oauth_token'][0], secret: params['oauth_token_secret'][0]}
    end
{'fraction_non_alphanumeric': 0.15490140040011433, 'fraction_numerical': 0.011717633609602743, 'mean_word_length': 3.847645429362881, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 17}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5787419', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1273, 'n_words': 216}