<commit_before>require 'nokogiri'
require 'open-uri'

module SeoParams

  class Yandex

    def initialize(url)
      url.match(/^(http:\/\/)/) ? @url = url : @url = 'http://' + url
    end


    def tic
      query = Nokogiri::XML(open("http://bar-navig.yandex.ru/u?ver=2&show=32&url=#{@url}"))
      tic = query.xpath('//@value')
      tic.to_s.to_i
    end

  end

end<commit_msg>Added function for checking pages in yandex index yap()<commit_after>require 'nokogiri'
require 'open-uri'

module SeoParams

  class Yandex

    def initialize(url)
      url.match(/^(http:\/\/)/) ? @url = url : @url = 'http://' + url
    end


    def tic
      query = Nokogiri::XML(open("http://bar-navig.yandex.ru/u?ver=2&show=32&url=#{@url}"))
      tic = query.xpath('//@value')
      tic.to_s.to_i
    end

    def yap
       pages = ask_yandex(@url)
      (pages.is_a? String) ? (@url = pages; pages = ask_yandex(pages); ) : pages
      pages
    end

    private
      def ask_yandex(url)

        doc = Nokogiri::HTML(open("http://webmaster.yandex.ua/check.xml?hostname=#{url}"))

        if doc.css('div.error-message').length > 0
          if doc.css('div.error-message').to_s.scan('Сайт является зеркалом')
            url_regexp = /\?hostname\=([a-zA-Z\.0-9]{1,})/
            new_url = doc.css('div.error-message').to_s.scan(url_regexp)[0][0]
            index = 'http://' + new_url
          end

        else
          doc.css('div.header div').each do |link|
              pages = /\d{1,}/
              @pages_in_index = link.content.scan(pages)
              index = @pages_in_index[0].to_i
          end
        end

        index
      end

  end

end
{'fraction_non_alphanumeric': 0.1683765841882921, 'fraction_numerical': 0.008449004224502113, 'mean_word_length': 2.128301886792453, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '205225', 'n_tokens_mistral': 641, 'n_tokens_neox': 616, 'n_words': 109}