<commit_before>require 'httparty'
require 'json'

module Biffbot
	class Base
		def initialize(token) 
			@token = token
		end
		def parse(url,options={})
			@url = url
			output = Hash.new
			request = "http://www.diffbot.com/api/article?token=#{@token}&url=#{@url}"
			options.each_pair do |key,value|
				if key.match(/html|dontStripAds|tags|comments|summary|stats/) && value == true
					request = request + "&#{key}"
				end
			end
			response = HTTParty.get(request)
			response.parsed_response.each_pair do |key,value|
				output[key.to_sym] = value
			end
			return output
		end
		def batch(urls, options={})
			output = []
			request = "http://www.diffbot.com/api/batch"
      batch = urls.map do |url|
        { :method => "GET", :relative_url => "/api/article?token=#{@token}&url=#{url}" }
      end
      options = { :body => {:token => @token, :batch => batch.to_json }, :basic_auth => @auth }
			response = HTTParty.post(request, options)
      
			JSON.parse(response.parsed_response).each do |response_dict|
        puts response_dict.inspect
				output << JSON.parse(response_dict["body"])
			end
			return output
    end
	end<commit_msg>declare urls as an array, otherwise method assumes 2nd url is part of options<commit_after>require 'httparty'
require 'json'

module Biffbot
	class Base
		def initialize(token) 
			@token = token
		end
		def parse(url,options={})
			@url = url
			output = Hash.new
			request = "http://www.diffbot.com/api/article?token=#{@token}&url=#{@url}"
			options.each_pair do |key,value|
				if key.match(/html|dontStripAds|tags|comments|summary|stats/) && value == true
					request = request + "&#{key}"
				end
			end
			response = HTTParty.get(request)
			response.parsed_response.each_pair do |key,value|
				output[key.to_sym] = value
			end
			return output
		end
		def batch(urls=[],options={})
			output = []
			request = "http://www.diffbot.com/api/batch"
      		batch = urls.map do |url|
    			{ :method => "GET", :relative_url => "/api/article?token=#{@token}&url=#{url}" }
      		end
     		options = { :body => {:token => @token, :batch => batch.to_json }, :basic_auth => @auth }
			response = HTTParty.post(request, options)
      
			JSON.parse(response.parsed_response).each do |response_dict|
        		puts response_dict.inspect
				output << JSON.parse(response_dict["body"])
			end
			return output
    	end
	end
{'fraction_non_alphanumeric': 0.1486770264594708, 'fraction_numerical': 0.00041999160016799666, 'mean_word_length': 3.2841726618705036, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313409', 'n_tokens_mistral': 915, 'n_tokens_neox': 812, 'n_words': 175}