<commit_before>## Salamander: A minimalistic ruby web crawling framework.
## Authored by: <NAME>

require 'time'
require 'thread'
require 'set'
require 'open-uri'
require 'openssl'

require 'json'
require 'open_uri_redirections'
require 'nokogiri'
require 'addressable/uri'

## Module
##     Salamander
## Description
##     The Crawler module provides an easy way for the other components of the Salamander system to perform crawling.
## Functions
##     Salamander::crawl
module Salamander

	## Function
	##     get_links
	## Description
	##     Extracts outgoing links from the HTML pointed to by the given URL string.
	## Parameters
	##     url		-	The URL of the HTML page the function is extracting links from.
	##     html		-	The HTML data to extract links from.
	def self.get_links(url, html)
		# Initialize
		uri = Addressable::URI.parse(url)
		# Parse as HTML
		_html = Nokogiri::HTML(html)
		# Get all anchors
		_html.xpath('//a').each do |l|
			# Extract hyper link
			href = l['href']
			# Skip if hyper link does not exist
			if href == nil then
				next
			end
			# Convert hyper link to URI object
			link = Addressable::URI.parse(href)
			# Skip if hyper link is not HTTP 
			if link.scheme != nil && link.scheme != 'http' && link.scheme != 'https' then
				next
			end
			# Convert hyper link to absolute form
			if link.host == nil then
				link.host = uri.host
			end
			if link.scheme == nil then
				link.scheme = uri.scheme
			end
			if link.port == nil then
				link.port = uri.port
			end
			# Remove link fragment
			link.fragment = nil
			# Yield
			yield link
		end
	end
	
	## Function
	##     crawl
	## Description
	##     Performs a restricted, unauthenticated, breadth-first crawl of the target web asset.
	##     Function blocks until all threads terminate.
	## Parameters
	##     urls		-	Required. A list of strings containing the seed URLs.
	##     args		-	Optional. Default: {}. A hash containing optional arguments for the function.
	##         visit	-	Optional. Default: nil. A lambda which accepts a URL, and returns a boolean which tells the crawler if the URL should be visited.
	##         delay	-	Optional. Default: 1. A positive float indicating the number of seconds between requests in one thread.
	##         threads  -	Optional. Default: 1. A positive integer indicating the number of allowed simultaneous requests to the target web asset.
	##         agent	-	Optional. Default: "Mozilla/5.0 (MSIE 9.0; Windows NT 6.1; Trident/5.0)". The user-agent string to be used.
	def crawl(urls, args = {})<commit_msg>Formatted inline documentation format to follow YARD<commit_after># Salamander: A minimalistic ruby web crawling framework.
# Authored by: <NAME>

require 'time'
require 'thread'
require 'set'
require 'open-uri'
require 'openssl'

require 'json'
require 'open_uri_redirections'
require 'nokogiri'
require 'addressable/uri'

# The module containing the Salamander framework itself.
module Salamander

	# Extracts outgoing links from the HTML pointed to by the given URL string.
	# @param url The URL of the HTML page the function is extracting links from.
	# @param html The HTML data to extract links from.
	def self.get_links(url, html)
		# Initialize
		uri = Addressable::URI.parse(url)
		# Parse as HTML
		_html = Nokogiri::HTML(html)
		# Get all anchors
		_html.xpath('//a').each do |l|
			# Extract hyper link
			href = l['href']
			# Skip if hyper link does not exist
			if href == nil then
				next
			end
			# Convert hyper link to URI object
			link = Addressable::URI.parse(href)
			# Skip if hyper link is not HTTP 
			if link.scheme != nil && link.scheme != 'http' && link.scheme != 'https' then
				next
			end
			# Convert hyper link to absolute form
			if link.host == nil then
				link.host = uri.host
			end
			if link.scheme == nil then
				link.scheme = uri.scheme
			end
			if link.port == nil then
				link.port = uri.port
			end
			# Remove link fragment
			link.fragment = nil
			# Yield
			yield link
		end
	end
	
	# Performs a restricted, unauthenticated, breadth-first crawl of the target web asset.
	# Function blocks until all threads terminate.
	# Optional Arguments (Place these inside the 'args' hash)
	#     visit:   A lambda which accepts a URL, and returns a boolean which tells the crawler if the URL should be visited.
	#     delay:   A positive float indicating the number of seconds between requests in one thread. Defaults to 1.
	#     threads: A positive integer indicating the number of allowed simultaneous requests to the target web asset. Defaults to 1.
	#     agent:   The user-agent string to be used. Defaults to "Mozilla/5.0 (MSIE 9.0; Windows NT 6.1; Trident/5.0)".
	# @param urls A list of strings containing the seed URLs.
	# @param args A hash containing optional arguments for the function.
	def crawl(urls, args = {})
{'fraction_non_alphanumeric': 0.0845771144278607, 'fraction_numerical': 0.0041459369817578775, 'mean_word_length': 3.1098807495741054, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5150407', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1412, 'n_words': 608}