<commit_before># encoding: utf-8
require 'cgi'
require 'open-uri'
require 'nokogiri'
require 'timeout'
require 'chinese_vocab/core_ext/array'
require 'with_validations'
require 'chinese_vocab/modules/helper_methods'

module Chinese
  class Scraper
    include WithValidations
    include HelperMethods

    attr_reader   :source, :word
    attr_accessor :sentences

    Sources = {
    nciku:
    {:url         => "http://www.nciku.com/search/all/examples/",
     :parent_sel  => "div.examples_box > dl",
     :cn_sel      => "//dt/span[1]",
     :en_sel      => "//dd/span[@class='tc_sub']",
                     # Only cn/en sentence pairs where the second node has a class 'tc_sub' belong together.
     :select_pair => lambda { |node1,node2| node1['class'] != "tc_sub" && node2['class'] == "tc_sub" },
                     # Just return the text stored in the node. :text_sel is mainly intended for jukuu (see below)
     :text_sel    => "text()",
                     # We want cn first, en second, but nciku does not return cn/en sentence pairs in a strict order.
     :reorder     => lambda { |text1,text2| if is_unicode?(text2) then [text2,text1] else [text1,text2] end }},
     jukuu:
     {:url         => "http://www.jukuu.com/search.php?q=",
      :parent_sel  => "table#Table1 table[width = '680']",
      :cn_sel      => "//tr[@class='c']",
      :en_sel      => "//tr[@class='e']",
                     # Only cn/en sentence pairs where the first node has a class 'e' belong together.
      :select_pair => lambda { |node1,node2| node1['class'] == "e" && node2['class'] != "e" },
      :text_sel    => "td[2]",
      :reorder     => lambda { |text1,text2| [text2,text1] }}
  }

  OPTIONS =  {:source =>  [:nciku,  lambda {|value| Sources.keys.include?(value) }],
              :size   =>  [:average, lambda {|value| [:short, :average, :long].include?(value) }]}


    # Options:
    # size => [:short, :average, :long], default = :average
    def self.sentences(word, options={})
      download_source = validate { :source }

      source = Sources[download_source]

      CGI.accept_charset = 'UTF-8'
      # Note: Use + because << changes the object on its left hand side, but + doesn't:
      # http://stackoverflow.com/questions/377768/string-concatenation-and-ruby/378258#378258
      url       = source[:url] + CGI.escape(word)
      # http://ruby-doc.org/stdlib-1.9.2/libdoc/timeout/rdoc/Timeout.html#method-c-timeout<commit_msg>changed the default size from :average to :short<commit_after># encoding: utf-8
require 'cgi'
require 'open-uri'
require 'nokogiri'
require 'timeout'
require 'chinese_vocab/core_ext/array'
require 'with_validations'
require 'chinese_vocab/modules/helper_methods'

module Chinese
  class Scraper
    include WithValidations
    include HelperMethods

    attr_reader   :source, :word
    attr_accessor :sentences

    Sources = {
    nciku:
    {:url         => "http://www.nciku.com/search/all/examples/",
     :parent_sel  => "div.examples_box > dl",
     :cn_sel      => "//dt/span[1]",
     :en_sel      => "//dd/span[@class='tc_sub']",
                     # Only cn/en sentence pairs where the second node has a class 'tc_sub' belong together.
     :select_pair => lambda { |node1,node2| node1['class'] != "tc_sub" && node2['class'] == "tc_sub" },
                     # Just return the text stored in the node. :text_sel is mainly intended for jukuu (see below)
     :text_sel    => "text()",
                     # We want cn first, en second, but nciku does not return cn/en sentence pairs in a strict order.
     :reorder     => lambda { |text1,text2| if is_unicode?(text2) then [text2,text1] else [text1,text2] end }},
     jukuu:
     {:url         => "http://www.jukuu.com/search.php?q=",
      :parent_sel  => "table#Table1 table[width = '680']",
      :cn_sel      => "//tr[@class='c']",
      :en_sel      => "//tr[@class='e']",
                     # Only cn/en sentence pairs where the first node has a class 'e' belong together.
      :select_pair => lambda { |node1,node2| node1['class'] == "e" && node2['class'] != "e" },
      :text_sel    => "td[2]",
      :reorder     => lambda { |text1,text2| [text2,text1] }}
  }

  OPTIONS =  {:source =>  [:nciku,  lambda {|value| Sources.keys.include?(value) }],
              :size   =>  [:short, lambda {|value| [:short, :average, :long].include?(value) }]}


    # Options:
    # size => [:short, :average, :long], default = :average
    def self.sentences(word, options={})
      download_source = validate { :source }

      source = Sources[download_source]

      CGI.accept_charset = 'UTF-8'
      # Note: Use + because << changes the object on its left hand side, but + doesn't:
      # http://stackoverflow.com/questions/377768/string-concatenation-and-ruby/378258#378258
      url       = source[:url] + CGI.escape(word)
      # http://ruby-doc.org/stdlib-1.9.2/libdoc/timeout/rdoc/Timeout.html#method-c-timeout
{'fraction_non_alphanumeric': 0.1564336372847011, 'fraction_numerical': 0.019452887537993922, 'mean_word_length': 2.72809667673716, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4598299', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1639, 'n_words': 394}