<commit_before>require 'nokogiri'
require 'ostruct'

module Anemone
  class Page

    # The URL of the page
    attr_reader :url
    # Headers of the HTTP response
    attr_reader :headers
    
    # OpenStruct for user-stored data
    attr_accessor :data
    # Integer response code of the page
    attr_accessor :code	
    # Array of redirect-aliases for the page
    attr_accessor :aliases
    # Boolean indicating whether or not this page has been visited in PageHash#shortest_paths!
    attr_accessor :visited
    # Depth of this page from the root of the crawl. This is not necessarily the
    # shortest path; use PageHash#shortest_paths! to find that value.
    attr_accessor :depth
    # URL of the page that brought us to this page
    attr_accessor :referer
    # Response time of the request for this page in milliseconds
    attr_accessor :response_time
    
    #
    # Create a new page
    #
    def initialize(url, body = nil, code = nil, headers = nil, aka = nil, referer = nil, depth = 0, response_time = nil)
      @url = url
      @code = code
      @headers = headers
      @headers['content-type'] ||= ['']
      @aliases = Array(aka)
      @data = OpenStruct.new
      @referer = referer
      @depth = depth || 0
      @response_time = response_time
      @body = body
    end
    
    # Nokogiri document for the HTML body
    def doc
      @doc ||= @body && html? && Nokogiri::HTML(@body)
    end
    
    # Array of distinct A tag HREFs from the page
    def links
      @links ||= begin
        if doc
          # get a list of distinct links on the page, in absolute url form
          links = doc.css('a[href]').inject([]) do |list, link|
            href = link.attributes['href'].content
            unless href.nil? or href.empty?
              url = to_absolute(href)
              list << url if in_domain?(url)
            end
            list
          end
          
          links.uniq!
          links
        else
          []
        end
      end
    end
    
    def discard_document!
      links # force parsing of page links before we trash the document
      @body = @doc = nil
    end
    
    #
    # Return a new page with the same *response* and *url*, but
    # with a 200 response code
    #    
    def alias_clone(url)
      p = clone
	  p.add_alias!(@aka) if !@aka.nil?
	  p.code = 200
	  p<commit_msg>refactor Page#links<commit_after>require 'nokogiri'
require 'ostruct'

module Anemone
  class Page

    # The URL of the page
    attr_reader :url
    # Headers of the HTTP response
    attr_reader :headers
    
    # OpenStruct for user-stored data
    attr_accessor :data
    # Nokogiri document for the HTML body
    attr_accessor :doc
    # Integer response code of the page
    attr_accessor :code	
    # Array of redirect-aliases for the page
    attr_accessor :aliases
    # Boolean indicating whether or not this page has been visited in PageHash#shortest_paths!
    attr_accessor :visited
    # Depth of this page from the root of the crawl. This is not necessarily the
    # shortest path; use PageHash#shortest_paths! to find that value.
    attr_accessor :depth
    # URL of the page that brought us to this page
    attr_accessor :referer
    # Response time of the request for this page in milliseconds
    attr_accessor :response_time
    
    #
    # Create a new page
    #
    def initialize(url, body = nil, code = nil, headers = nil, aka = nil, referer = nil, depth = 0, response_time = nil)
      @url = url
      @code = code
      @headers = headers
      @headers['content-type'] ||= ['']
      @aliases = Array(aka)
      @data = OpenStruct.new
      @referer = referer
      @depth = depth || 0
      @response_time = response_time
      @doc = Nokogiri::HTML(body) if body && html? rescue nil
    end

    # Array of distinct A tag HREFs from the page
    def links
      return @links unless @links.nil?
      @links = []
      return @links if !doc
      
      doc.css('a').each do |a|
        u = a.attributes['href'].content rescue nil
        next if u.nil? or u.empty?
        abs = to_absolute(URI(u)) rescue next
        @links << abs if in_domain?(abs)
      end
      @links.uniq!
      @links
    end
    
    def discard_document!
      links # force parsing of page links before we trash the document
      @doc = nil
    end
    
    #
    # Return a new page with the same *response* and *url*, but
    # with a 200 response code
    #    
    def alias_clone(url)
      p = clone
	  p.add_alias!(@aka) if !@aka.nil?
	  p.code = 200
	  p
{'fraction_non_alphanumeric': 0.07775330396475771, 'fraction_numerical': 0.003524229074889868, 'mean_word_length': 2.1447368421052633, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1261456', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1356, 'n_words': 505}