<commit_before>require 'nokogiri'
require 'open-uri'

module OxfordLearnersDictionaries
  class English
    attr_reader :definition, :type, :urls, :word

    def initialize word
      formatted_word = word.strip.gsub(' ', '-') rescue ''
      param_word = formatted_word.gsub('-', '+')
      main_url =  "http://www.oxfordlearnersdictionaries.com/definition/english/#{formatted_word}?q=#{param_word}"

      @urls = [ main_url, main_url.gsub('?q=', '1?q=') ]
      @word = formatted_word
      @definition = Hash.new
    end

    def look_up
      @urls.each do |url|
        begin
          @page = Nokogiri::HTML(open(url))
          break
        rescue OpenURI::HTTPError
          @page = nil
        end
      end
      return {} if @page.nil?

      @page.css('.idm-gs').remove
      parse
      self.definition
    end

    private
    def parse
      parse_type
      if @page.css('.num').count > 0
        parse_multiple_definitions
      else
        parse_single_definition
      end
      self
    end

    def parse_type
      @type = @page.css('.pos').first.text rescue 'unknown'
    end

    def parse_single_definition
      definitions =  @page.css('.def')
      @definition[:definition_0] = definitions.count > 0 ? definitions[0].text : definitions.text
    end

    def parse_multiple_definitions
      @page.css('.num').count.times do |index|
        @definition["definition_#{index}".to_sym] = @page.css('.def')[index].text
      end
    end
  end
end

# definition.css(".x-g").each do |examples|
#   puts "# #{examples.css(".x").text}"
# end<commit_msg>Add examples as simple hash<commit_after>require 'nokogiri'
require 'open-uri'

module OxfordLearnersDictionaries
  class English
    attr_reader :definition, :type, :urls, :word, :examples

    def initialize word
      formatted_word = word.strip.gsub(' ', '-') rescue ''
      param_word = formatted_word.gsub('-', '+')
      main_url =  "http://www.oxfordlearnersdictionaries.com/definition/english/#{formatted_word}?q=#{param_word}"

      @urls = [ main_url, main_url.gsub('?q=', '1?q=') ]
      @word = formatted_word
      @definition = Hash.new
      @examples = Hash.new
    end

    def look_up
      @urls.each do |url|
        begin
          @page = Nokogiri::HTML(open(url))
          break
        rescue OpenURI::HTTPError
          @page = nil
        end
      end
      return {} if @page.nil?

      @page.css('.idm-gs').remove
      parse
      self.definition
    end

    private
    def parse
      parse_type
      parse_examples
      if @page.css('.num').count > 0
        parse_multiple_definitions
      else
        parse_single_definition
      end
      self
    end

    def parse_type
      @type = @page.css('.pos').first.text rescue 'unknown'
    end

    def parse_single_definition
      definitions =  @page.css('.def')
      @definition[:definition_0] = definitions.count > 0 ? definitions[0].text : definitions.text
    end

    def parse_multiple_definitions
      @page.css('.num').count.times do |index|
        @definition[:"definition_#{index}"] = @page.css('.def')[index].text
      end
    end

    def parse_examples
      @page.css(".x-g").each_with_index do |example, index|
        @examples[:"examples_#{index}"] = example.text.strip.capitalize
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12897366030881016, 'fraction_numerical': 0.003027550711474417, 'mean_word_length': 2.406185567010309, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '242004', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1042, 'n_words': 213}