<commit_before>require File.dirname(__FILE__) + '/../lib/http-machine.rb'
require 'sax-machine'

class YahooBOSS < HTTPMachine::Remote  
  API_ID = "fSMqdK_V34H9bI8GtgrxKWE9WAnX9QFk26Lakdvbi35k1wTzRZ<KEY>
  
  remote_server "http://boss.yahooapis.com"
  remote_method :web_search,  {
    :resource => "ysearch/web/v1",
    :method => :get, 
    :params => {:appid => API_ID, :format => "xml", :view => "keyterms", :abstract => "long", :count => 100},
    :response_handler => :parse }  
  remote_method :site_search, {
    :resource => "ysearch/se_inlink/v1/", 
    :method => :get, 
    :params => {:appid => API_ID, :format => "xml"}, 
    :response_handler => :parse }

  include SAXMachine
  class Result
    include SAXMachine
    element :abstract
    element :date
    element :title
    element :url
    elements :term, :as => :terms
    
    def date=(value)
      @date = Date.parse(value)
    end
  end
  elements :result, :as => :results, :class => Result
  
  element :nextpage, :as => :next_page
  element :resultset_web,       :value => :totalhits, :as => :total_hits<commit_msg>removed boss key<commit_after>require File.dirname(__FILE__) + '/../lib/http-machine.rb'
require 'sax-machine'

class YahooBOSS < HTTPMachine::Remote  
  API_ID = "..."
  
  remote_server "http://boss.yahooapis.com"
  remote_method :web_search,  {
    :resource => "ysearch/web/v1",
    :method => :get, 
    :params => {:appid => API_ID, :format => "xml", :view => "keyterms", :abstract => "long", :count => 100},
    :response_handler => :parse }  
  remote_method :site_search, {
    :resource => "ysearch/se_inlink/v1/", 
    :method => :get, 
    :params => {:appid => API_ID, :format => "xml"}, 
    :response_handler => :parse }

  include SAXMachine
  class Result
    include SAXMachine
    element :abstract
    element :date
    element :title
    element :url
    elements :term, :as => :terms
    
    def date=(value)
      @date = Date.parse(value)
    end
  end
  elements :result, :as => :results, :class => Result
  
  element :nextpage, :as => :next_page
  element :resultset_web,       :value => :totalhits, :as => :total_hits
{'fraction_non_alphanumeric': 0.15700934579439252, 'fraction_numerical': 0.009813084112149532, 'mean_word_length': 3.248015873015873, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179371', 'n_tokens_mistral': 778, 'n_tokens_neox': 751, 'n_words': 168}