<commit_before>#!/usr/bin/env ruby
# encoding: utf-8

require 'nokogiri'
require 'typhoeus'
require 'sequel'
require 'uri'
require './lib/valid'
require './cli'
require './options'

class String
  def numeric?
    !self.match(/[^0-9]/)
  end

  def alpha?
    !self.match(/[^A-z]+$/)
  end
end

class Database

  def on_start
    to_crawl = queue.where(:in_use => 1)
    to_crawl.each { |item| queue.where(:id => item[:id]).update(:in_use => 0) }
  end

  # For each page in the urls table, see if the page has been
  # accessed within the seed time. If it has not,add that url to the
  # queue with the pattern set to '' and force set to 0.
  #
  def seed
    to_crawl = urls.where{ accessed < Time.now - @@SHELF }
    to_crawl.each { |row| insert_data_into_into(queue, [row[:url], '', 0]) }
    return crawl_queue unless queue.empty?
  end

  # Get the first row in the queue and check to see if either the force flag is
  # on, or if the url has not been added to the urls table, or if it has been
  # added to the urls table AND the url has not been accessed within the seed
  # time
  #
  def crawl_queue
    # Get the first item in the queue table. Variable 'row' gets that value.
    row = queue.first

    # If the current row is in use, get the next row that is not in use.
    if row[:in_use] == 1
      row = queue.where(:in_use => 0).limit(1).first
    end
    # Set the current row to in use
    queue.where(:id => row[:id]).update(:in_use => 1)

    # If the force column in the sequel object is set to true crawl the page.
    # Elsif the url of the sequel object is not in the urls table crawl the page.
    # Elsif the sequel object is in the urls table, but has not been accessed
    # within the shelf time crawl the page.
    if row[:force] == 1
      crawl_url(row[:id])
    elsif !urls[:url => row[:url]]
      crawl_url(row[:id])
    elsif urls[:url => row[:url]] && Time.now - Time.parse(urls.where(:url => row[:url]).get(:accessed)) > @@SHELF
      crawl_url(row[:id])
    end
    # Delete the sequel object from the queue table.
    queue.where(:id => row[:id]).delete
  end

  def crawl_url(queue_id)

    item = queue[:id => queue_id]
    url = item[:url]
    last_accessed = Time.now
    parsed_links = []
    old_links = []
    type_array = []

    if url[-1, 10] == "index.html"
      url = url[0..-10]
    end

    response = Typhoeus::Request.get(url,
                                    :timeout_ms => 30000,
                                    :followlocation => true,
                                    :maxredirs => 5,
                                    :headers => {
                                      'User-Agent' => "Cosa/0.2 ()"
                                    })
    url = response.effective_url
    url[0..4].downcase
    response_time = response.total_time.round(6)

    content_type = response.headers_hash["Content-Type"]
    if response.headers_hash["Content-Length"].to_s.numeric?
      content_length = response.headers_hash["Content-Length"]
    else
      content_length = ""
    end

    if content_type.include?(';')
      content_type = content_type[0..content_type.index(';')-1]
    elsif content_type == {}
      content_type = ""
    end

    status = response.code
    body = response.body
    if status == 404
      valid = [0, 0]
    else
      valid = valid?(url, content_type)
      # valid = [1, 1]
    end

    if valid[1] == "html"
      if url.include?(domain)

        # Iterate through the a, link, img, video, audio, script, and object elements on the page.
        Nokogiri::HTML(body).css('a', 'link', 'img', 'video', 'audio', 'script', 'object').each do |item|
          if item.name == "link"<commit_msg>Fixed another NoMethodError crash.<commit_after>#!/usr/bin/env ruby
# encoding: utf-8

require 'nokogiri'
require 'typhoeus'
require 'sequel'
require 'uri'
require './lib/valid'
require './cli'
require './options'

class String
  def numeric?
    !self.match(/[^0-9]/)
  end

  def alpha?
    !self.match(/[^A-z]+$/)
  end
end

class Database

  def on_start
    to_crawl = queue.where(:in_use => 1)
    to_crawl.each { |item| queue.where(:id => item[:id]).update(:in_use => 0) }
  end

  # For each page in the urls table, see if the page has been
  # accessed within the seed time. If it has not,add that url to the
  # queue with the pattern set to '' and force set to 0.
  #
  def seed
    to_crawl = urls.where{ accessed < Time.now - @@SHELF }
    to_crawl.each { |row| insert_data_into_into(queue, [row[:url], '', 0]) }
    return crawl_queue unless queue.empty?
  end

  # Get the first row in the queue and check to see if either the force flag is
  # on, or if the url has not been added to the urls table, or if it has been
  # added to the urls table AND the url has not been accessed within the seed
  # time
  #
  def crawl_queue
    # Get the first item in the queue table. Variable 'row' gets that value.
    row = queue.first

    # If the current row is in use, get the next row that is not in use.
    if row[:in_use] == 1
      row = queue.where(:in_use => 0).limit(1).first
    end
    # Set the current row to in use
    queue.where(:id => row[:id]).update(:in_use => 1)

    # If the force column in the sequel object is set to true crawl the page.
    # Elsif the url of the sequel object is not in the urls table crawl the page.
    # Elsif the sequel object is in the urls table, but has not been accessed
    # within the shelf time crawl the page.
    if row[:force] == 1
      crawl_url(row[:id])
    elsif !urls[:url => row[:url]]
      crawl_url(row[:id])
    elsif urls[:url => row[:url]] && Time.now - Time.parse(urls.where(:url => row[:url]).get(:accessed)) > @@SHELF
      crawl_url(row[:id])
    end
    # Delete the sequel object from the queue table.
    queue.where(:id => row[:id]).delete
  end

  def crawl_url(queue_id)

    item = queue[:id => queue_id]
    url = item[:url]
    last_accessed = Time.now
    parsed_links = []
    old_links = []
    type_array = []

    if url[-1, 10] == "index.html"
      url = url[0..-10]
    end

    response = Typhoeus::Request.get(url,
                                    :timeout_ms => 30000,
                                    :followlocation => true,
                                    :maxredirs => 5,
                                    :headers => {
                                      'User-Agent' => "Cosa/0.2 ()"
                                    })
    url = response.effective_url
    url[0..4].downcase
    response_time = response.total_time.round(6)
    begin
      content_type = response.headers_hash["Content-Type"]
      if response.headers_hash["Content-Length"].to_s.numeric?
        content_length = response.headers_hash["Content-Length"]
      else
        content_length = ""
      end
    rescue NoMethodError
      content_type = "NA"
      content_length = "NA"
    end



    if content_type.include?(';')
      content_type = content_type[0..content_type.index(';')-1]
    elsif content_type == {}
      content_type = ""
    end

    status = response.code
    body = response.body
    if status == 404
      valid = [0, 0]
    else
      valid = valid?(url, content_type)
      # valid = [1, 1]
    end

    if valid[1] == "html"
      if url.include?(domain)

        # Iterate through the a, link, img, video, audio, script, and object elements on the page.
        Nokogiri::HTML(body).css('a', 'link', 'img', 'video', 'audio', 'script', 'object').each do |item|
          if item.name == "link"
{'fraction_non_alphanumeric': 0.12355057976809276, 'fraction_numerical': 0.010395841663334666, 'mean_word_length': 2.251299826689775, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147968', 'n_tokens_mistral': 2493, 'n_tokens_neox': 2374, 'n_words': 768}