<commit_before>#!/usr/bin/env ruby
#
require "rubygems"
require "bundler/setup"

#require "httmultiparty"
require "httparty"
require "base64"
require "time"
require "yaml"


CONFIG = YAML.load_file("config.yml") unless defined? CONFIG

class FetchEveryone
  include HTTParty
  #base_uri = "https://api.fetcheveryone.com" # This defaults to http for some reason :-( so can't use

  def initialize(k, s)
    authorization = Base64.strict_encode64("#{k}:#{s}")
    response = self.class.post(
      'https://api.fetcheveryone.com/token.php',
      headers: {'Accept' => 'application/json', 'Authorization' => "Basic #{authorization}"}
    )
    @access_token = response['access_token']
  end

  def getResource(resource)
    response = self.class.get(
      "https://api.fetcheveryone.com/api.php?request=#{resource}",
      headers: {'Authorization' => "Bearer #{@access_token}", 'Content-type' => 'application/json'}
    )
    puts response.body
  end

end

fe = FetchEveryone.new("#{CONFIG['api_key']}", "#{CONFIG['api_secret']}")
fe.getResource("forum/threads")<commit_msg>Added support for options and optimized things a little<commit_after>#!/usr/bin/env ruby
#
require "rubygems"
require "bundler/setup"

#require "httmultiparty"
require "httparty"
require "base64"
require "time"
require "yaml"

CONFIG = YAML.load_file("config.yml") unless defined? CONFIG

class FetchEveryone
  include HTTParty
  #base_uri = "https://api.fetcheveryone.com" # This defaults to http for some reason :-( so can't use

  def initialize(k, s)
    authorization = Base64.strict_encode64("#{k}:#{s}")
    response = self.class.post(
      'https://api.fetcheveryone.com/token.php',
      headers: {'Accept' => 'application/json', 'Authorization' => "Basic #{authorization}"}
    )
    @access_token = response['access_token']
  end

  def getResource(resource, options = nil)
    encoded_opts = "&#{URI.encode_www_form(options)}" unless options.nil?
    response = self.class.get(
      "https://api.fetcheveryone.com/api.php?request=#{resource}#{encoded_opts}",
      headers: {'Authorization' => "Bearer #{@access_token}", 'Content-type' => 'application/json'},
    )
    puts response.body
  end

end

fe = FetchEveryone.new("#{CONFIG['api_key']}", "#{CONFIG['api_secret']}")
#fe.getResource("forum/threads")
fe.getResource("forum/threads", {category: "training", items: 5})
{'fraction_non_alphanumeric': 0.16476069462092333, 'fraction_numerical': 0.005506141465480729, 'mean_word_length': 4.637231503579952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1958293', 'n_tokens_mistral': 800, 'n_tokens_neox': 774, 'n_words': 164}