<commit_before>require 'open-uri'
require 'nokogiri'

class Robot
  def initialize(base)
    http = 'http://'
    base = http + base unless base =~ /^#{ http }/

    @base = URI.parse(base)
    @visited_uris = Array.new

    parse(@base)
  end

  def parse(uri)
    puts "Parsing: #{ uri }"
    @visited_uris << uri

    begin
      doc = Nokogiri::HTML(open(uri))
    rescue
      return
    end

    uris = doc.xpath("//a")
    
    # Wordpress OpenID plugin
    openids =
      # Select <a class="openid_link">
      uris.select{ |u|
        c = u.attributes['class']
        c.present? && c.value.include?('openid_link')
      }.map{ |e|
        e.attributes['href'].try(:value)
      }

    # LiveJournal
    openids |= 
      # Select <span class="ljuser">..<img src="..openid">..<a rel="nofollow"></span>
      doc.css("span.ljuser").select{ |s|
        s.xpath("//img").select{ |i|
          src = i.attributes['src']
          src.present? && src.value.include?('openid')
        }.any?
      }.map{ |s|
        # Select nofollow href
        s.xpath("//a").select{ |a|
          rel = a.attributes['rel']
          rel.present? && rel.value.include?('nofollow')
        }.map{ |l|
          l.attributes['href'].try(:value)
        }
      }.flatten.compact

    # Save found OpenIDs
    openids.each do |id|
      puts "OpenID found: #{ id } "
      u = Uri.find_or_create_by_uri(id)
      next if u.new_record?
      u.refresh!
    end


    uris.map{ |u| u.attributes['href'].try(:value) }.compact.each do |u|
      begin<commit_msg>Add support for Stackoverflow blog. Enhance robot<commit_after>require 'open-uri'
require 'nokogiri'

class Robot
  def initialize(base)
    http = 'http://'
    base = http + base unless base =~ /^#{ http }/

    @base = URI.parse(base)
    @visited_uris = Array.new
    @visited_openids = Array.new

    parse(@base)
  end

  def parse(uri)
    puts "Parsing: #{ uri }"
    @visited_uris << uri

    begin
      doc = Nokogiri::HTML(open(uri))
    rescue
      return
    end

    uris = doc.xpath("//a")
    
    # Wordpress OpenID plugin
    openids =
      # Select <a class="openid_link">
      uris.select{ |u|
        c = u.attributes['class']
        c.present? && c.value.include?('openid_link')
      }.map{ |e|
        e.attributes['href'].try(:value)
      }

    # LiveJournal
    # livejournal.com
    openids |= 
      # Select <span class="ljuser">..<img src="..openid">..<a rel="nofollow"></span>
      doc.css("span.ljuser").select{ |s|
        s.xpath("//img").select{ |i|
          src = i.attributes['src']
          src.present? && src.value.include?('openid')
        }.any?
      }.map{ |s|
        # Select nofollow href
        s.xpath("//a").select{ |a|
          rel = a.attributes['rel']
          rel.present? && rel.value.include?('nofollow')
        }.map{ |l|
          l.attributes['href'].try(:value)
        }
      }.flatten.compact

    # StackOverflow Blog
    # http://blog.stackoverflow.com/
    openids |=
      doc.css("ol.commentlist a.url").map{ |a|
        a.attributes['href'].try(:value)
      }
      
    # Save found OpenIDs
    openids.each do |id|
      puts "OpenID: #{ id }"

      if @visited_openids.include?(id)
        puts "        repeated."
        next
      end

      @visited_openids << id

      if Uri.find_by_uri(id)
        puts "        already saved."
        next
      end

      u = Uri.create(:uri => id)

      if u.new_record?
        puts "        invalid."
        next
      end

      puts "        added."
      u.refresh!
    end


    uris.map{ |u| u.attributes['href'].try(:value) }.compact.each do |u|
      begin
{'fraction_non_alphanumeric': 0.1620510008225939, 'fraction_numerical': 0.0, 'mean_word_length': 1.8952380952380952, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7407806', 'n_tokens_mistral': 1286, 'n_tokens_neox': 1255, 'n_words': 252}