<commit_before>require 'uri'
require 'cgi'

module WikipediaApi
  USER_AGENT = 'DbpediaLite/1'
  API_URI = URI.parse('http://en.wikipedia.org/w/api.php')
  ABSTRACT_MAX_LENGTH = 500
  HTTP_TIMEOUT = 5

  def self.page_info(args)
    data = self.get('query', {:redirects => 1, :prop => 'info'}.merge(args))

    unless data['query'].nil? or data['query']['pages'].empty?
      info = data['query']['pages'].values.first
      return info unless info.has_key?('missing')
    end
  end

  def self.search(query, args={})
    data = self.get('query', {:list => 'search', :prop => 'info', :srsearch => query}.merge(args))

    data['query']['search']
  end

  def self.get(action, args={})
    items = []
    args.merge!(:action => action, :format => 'json')

    keys = args.keys.sort {|a,b| a.to_s <=> b.to_s}
    keys.each do |key|
     items << CGI::escape(key.to_s)+'='+CGI::escape(args[key].to_s)
    end

    uri = API_URI.clone
    uri.query = items.join('&')
    res = Net::HTTP.start(uri.host, uri.port) do |http|
      http.read_timeout = HTTP_TIMEOUT
      http.open_timeout = HTTP_TIMEOUT
      http.get(uri.request_uri, {'User-Agent' => USER_AGENT})
    end

    # Throw exception if unsuccessful
    res.value

    JSON.parse(res.body)
  end

  def self.category_members(title, args={})
    # FIXME: this should use pageid, when it is available in the MediaWiki API
    data = self.get('query', {
      :list => 'categorymembers',
      :cmprop => 'ids|title',
      :cmsort => 'sortkey',
      :cmtitle => title,<commit_msg>FIXME: throw an exception if Wikipedia reports a problem<commit_after>require 'uri'
require 'cgi'

module WikipediaApi
  USER_AGENT = 'DbpediaLite/1'
  API_URI = URI.parse('http://en.wikipedia.org/w/api.php')
  ABSTRACT_MAX_LENGTH = 500
  HTTP_TIMEOUT = 5

  def self.page_info(args)
    data = self.get('query', {:redirects => 1, :prop => 'info'}.merge(args))

    unless data['query'].nil? or data['query']['pages'].empty?
      info = data['query']['pages'].values.first
      return info unless info.has_key?('missing')
    end
  end

  def self.search(query, args={})
    data = self.get('query', {:list => 'search', :prop => 'info', :srsearch => query}.merge(args))

    data['query']['search']
  end

  def self.get(action, args={})
    items = []
    args.merge!(:action => action, :format => 'json')

    keys = args.keys.sort {|a,b| a.to_s <=> b.to_s}
    keys.each do |key|
     items << CGI::escape(key.to_s)+'='+CGI::escape(args[key].to_s)
    end

    uri = API_URI.clone
    uri.query = items.join('&')
    res = Net::HTTP.start(uri.host, uri.port) do |http|
      http.read_timeout = HTTP_TIMEOUT
      http.open_timeout = HTTP_TIMEOUT
      http.get(uri.request_uri, {'User-Agent' => USER_AGENT})
    end

    # Throw exception if unsuccessful
    res.value

    JSON.parse(res.body)

    # FIXME: throw an exception if Wikipedia reports a problem
  end

  def self.category_members(title, args={})
    # FIXME: this should use pageid, when it is available in the MediaWiki API
    data = self.get('query', {
      :list => 'categorymembers',
      :cmprop => 'ids|title',
      :cmsort => 'sortkey',
      :cmtitle => title,
{'fraction_non_alphanumeric': 0.16902738432483475, 'fraction_numerical': 0.003777148253068933, 'mean_word_length': 3.260053619302949, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2807811', 'n_tokens_mistral': 1127, 'n_tokens_neox': 1092, 'n_words': 248}