<commit_before>require 'excon'
require 'nokogiri'

class Metanym
  VERSION = "0.1.0"
  URL = "http://thesaurus.com/browse/"

  def initialize(query)
    @url = URI.encode(URL + query)
  end

  def synonyms
    @synonyms ||= items_at 'Synonyms'
  end

  def antonyms
    @antonyms ||= items_at 'Antonyms'
  end

  def definition
    @definition ||= doc.at_xpath('//td[.="Definition:"]/following::td').text
  rescue NoMethodError
    nil
  end

  def items_at(head)
    doc.at_xpath("//td[.='#{head}:']/following-sibling::td").text.strip.split(/[\n,]\s*/)
  rescue NoMethodError
    []
  end

  def doc
    @doc ||= Nokogiri.HTML source
  end

  def source
    Excon.get(@url, :expects => 200).body
  end<commit_msg>Masquerade as a WebKit/Chrome like browser.<commit_after>require 'excon'
require 'nokogiri'

class Metanym
  VERSION = "0.1.0"
  URL = "http://thesaurus.com/browse/"

  def initialize(query)
    @url = URI.encode(URL + query)
  end

  def synonyms
    @synonyms ||= items_at 'Synonyms'
  end

  def antonyms
    @antonyms ||= items_at 'Antonyms'
  end

  def definition
    @definition ||= doc.at_xpath('//td[.="Definition:"]/following::td').text
  rescue NoMethodError
    nil
  end

  def items_at(head)
    doc.at_xpath("//td[.='#{head}:']/following-sibling::td").text.strip.split(/[\n,]\s*/)
  rescue NoMethodError
    []
  end

  def doc
    @doc ||= Nokogiri.HTML source
  end

  def source
    Excon.get(@url, :method => 'GET', :expects => 200, :headers => {"User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13"}).body
  end 
{'fraction_non_alphanumeric': 0.16625615763546797, 'fraction_numerical': 0.0166256157635468, 'mean_word_length': 3.198966408268734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7079440', 'n_tokens_mistral': 669, 'n_tokens_neox': 633, 'n_words': 129}