<commit_before><?php
/**
 *
 * Simple Machines Forum (SMF)
 * Simple cURL class to fetch a web page
 * Properly redirects even with safe mode and basedir restrictions
 * Can provide simple post options to a page
 *
 * Load class
 * Initiate as
 *  - $fetch_data = new cURL_fetch_web_data();
 *	- optionaly pass an array of cURL options and redirect count
 *	- cURL_fetch_web_data(cURL options array, Max redirects);
 *  - $fetch_data = new cURL_fetch_web_data(array(CURLOPT_SSL_VERIFYPEER => 1), 5);
 *
 * Make the call
 *  - $fetch_data('http://www.simplemachines.org'); // fetch a page
 *  - $fetch_data('http://www.simplemachines.org', array('user' => 'name', 'password' => 'password')); // post to a page
 *  - $fetch_data('http://www.simplemachines.org', parameter1&parameter2&parameter3); // post to a page
 *
 *
 * Get the data
 *  - $fetch_data->result('body'); // just the page content
 *  - $fetch_data->result(); // an array of results, body, header, http result codes
 *  - $fetch_data->result_raw(); // show all results of all calls (in the event of a redirect)
 *  - $fetch_data->result_raw(0); // show all results of call x
 *
 * @package SMF
 * @author Simple Machines http://www.simplemachines.org
 * @copyright 2011 Simple Machines
 * @license http://www.simplemachines.org/about/smf/license.php BSD
 *
 * @version 2.1 Alpha 1
 */

if (!defined('SMF'))
	die('Hacking attempt...');

class curl_fetch_web_data
{
	private $default_options = array(
		CURLOPT_RETURNTRANSFER	=> 1, // Get returned value as a string (don't output it)
		CURLOPT_HEADER			=> 1, // We need the headers to do our own redirect
		CURLOPT_FOLLOWLOCATION	=> 0, // Don't follow, we will do it ourselves so safe mode and open_basedir will dig it
		CURLOPT_USERAGENT		=> 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:11.0) Gecko Firefox/11.0', // set a normal looking useragent
		CURLOPT_CONNECTTIMEOUT	=> 15, // Don't wait forever on a connection
		CURLOPT_TIMEOUT			=> 90, // A page should load in this amount of time
		CURLOPT_MAXREDIRS		=> 5, // stop after this many redirects
		CURLOPT_ENCODING		=> 'gzip,deflate', // accept gzip and decode it
		CURLOPT_SSL_VERIFYPEER	=> 0, // stop cURL from verifying the peer's certificate
		CURLOPT_SSL_VERIFYHOST	=> 0, // stop cURL from verifying the peer's host
		CURLOPT_POST			=> 0, // no post data unless its passed
	);

	/**
	* Start the curl object
	* - allow for user override values
	*
	* @param type $options, cURL options as an array
	* @param type $max_redirect, use to overide the default of 3
	*/
	public function __construct($options = array(), $max_redirect = 3)
	{
		// Initialize class variables
		$this->max_redirect = intval($max_redirect);
		$this->user_options = $options;
	}

	/**
	* Main calling function, 
	*  - will request the page data from a given $url
	*  - optionally will post data to the page form if post data is supplied
	*  - passed arrays will be converted to a post string joined with &'s
	*  - calls set_options to set the curl opts array values based on the defaults and user input
	*
	* @param type $url, the site we are going to fetch
	* @param type $post_data, any post data as form name => value
	*/
	public function get_url_data($url, $post_data = array())
	{
		// POSTing some data perhaps?
		if (!empty($post_data) && is_array($post_data))
			$this->post_data = $this->build_post_data($post_data);
		elseif (!empty($post_data))
			$this->post_data = trim($post_data);

		// set the options and get it
		$this->set_options();
		$this->curl_request(str_replace(' ', '%20', $url));
	}

	/**
	* Makes the actual cURL call
	*  - stores responses (url, code, error, headers, body) in the response array
	*  - detects 301, 302, 307 codes and will redirect to the given response header location
	*
	* @param type $url, site to fetch
	* @param type $redirect, flag to indicate if this was a redirect request or not
	* @return boolean
	*/
	private function curl_request($url, $redirect = false)
	{
		// we do have a url I hope
		if ($url == '')
			return false;<commit_msg>A very small addition for chaining methods, like: $curl->get_url_data('http://www.simplemachines.org')->result('body');<commit_after><?php
/**
 *
 * Simple Machines Forum (SMF)
 * Simple cURL class to fetch a web page
 * Properly redirects even with safe mode and basedir restrictions
 * Can provide simple post options to a page
 *
 * Load class
 * Initiate as
 *  - $fetch_data = new cURL_fetch_web_data();
 *	- optionaly pass an array of cURL options and redirect count
 *	- cURL_fetch_web_data(cURL options array, Max redirects);
 *  - $fetch_data = new cURL_fetch_web_data(array(CURLOPT_SSL_VERIFYPEER => 1), 5);
 *
 * Make the call
 *  - $fetch_data('http://www.simplemachines.org'); // fetch a page
 *  - $fetch_data('http://www.simplemachines.org', array('user' => 'name', 'password' => 'password')); // post to a page
 *  - $fetch_data('http://www.simplemachines.org', parameter1&parameter2&parameter3); // post to a page
 *
 *
 * Get the data
 *  - $fetch_data->result('body'); // just the page content
 *  - $fetch_data->result(); // an array of results, body, header, http result codes
 *  - $fetch_data->result_raw(); // show all results of all calls (in the event of a redirect)
 *  - $fetch_data->result_raw(0); // show all results of call x
 *
 * @package SMF
 * @author Simple Machines http://www.simplemachines.org
 * @copyright 2011 Simple Machines
 * @license http://www.simplemachines.org/about/smf/license.php BSD
 *
 * @version 2.1 Alpha 1
 */

if (!defined('SMF'))
	die('Hacking attempt...');

class curl_fetch_web_data
{
	private $default_options = array(
		CURLOPT_RETURNTRANSFER	=> 1, // Get returned value as a string (don't output it)
		CURLOPT_HEADER			=> 1, // We need the headers to do our own redirect
		CURLOPT_FOLLOWLOCATION	=> 0, // Don't follow, we will do it ourselves so safe mode and open_basedir will dig it
		CURLOPT_USERAGENT		=> 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:11.0) Gecko Firefox/11.0', // set a normal looking useragent
		CURLOPT_CONNECTTIMEOUT	=> 15, // Don't wait forever on a connection
		CURLOPT_TIMEOUT			=> 90, // A page should load in this amount of time
		CURLOPT_MAXREDIRS		=> 5, // stop after this many redirects
		CURLOPT_ENCODING		=> 'gzip,deflate', // accept gzip and decode it
		CURLOPT_SSL_VERIFYPEER	=> 0, // stop cURL from verifying the peer's certificate
		CURLOPT_SSL_VERIFYHOST	=> 0, // stop cURL from verifying the peer's host
		CURLOPT_POST			=> 0, // no post data unless its passed
	);

	/**
	* Start the curl object
	* - allow for user override values
	*
	* @param type $options, cURL options as an array
	* @param type $max_redirect, use to overide the default of 3
	*/
	public function __construct($options = array(), $max_redirect = 3)
	{
		// Initialize class variables
		$this->max_redirect = intval($max_redirect);
		$this->user_options = $options;
	}

	/**
	* Main calling function, 
	*  - will request the page data from a given $url
	*  - optionally will post data to the page form if post data is supplied
	*  - passed arrays will be converted to a post string joined with &'s
	*  - calls set_options to set the curl opts array values based on the defaults and user input
	*
	* @param type $url, the site we are going to fetch
	* @param type $post_data, any post data as form name => value
	*/
	public function get_url_data($url, $post_data = array())
	{
		// POSTing some data perhaps?
		if (!empty($post_data) && is_array($post_data))
			$this->post_data = $this->build_post_data($post_data);
		elseif (!empty($post_data))
			$this->post_data = trim($post_data);

		// set the options and get it
		$this->set_options();
		$this->curl_request(str_replace(' ', '%20', $url));

		return $this;
	}

	/**
	* Makes the actual cURL call
	*  - stores responses (url, code, error, headers, body) in the response array
	*  - detects 301, 302, 307 codes and will redirect to the given response header location
	*
	* @param type $url, site to fetch
	* @param type $redirect, flag to indicate if this was a redirect request or not
	* @return boolean
	*/
	private function curl_request($url, $redirect = false)
	{
		// we do have a url I hope
		if ($url == '')
			return false;
{'fraction_non_alphanumeric': 0.12061591103507271, 'fraction_numerical': 0.011976047904191617, 'mean_word_length': 3.6659064994298745, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 11, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5191354', 'n_tokens_mistral': 2816, 'n_tokens_neox': 2632, 'n_words': 937}