<commit_before><?php
/**
 * Acces Class
 *
 * @author     <NAME>
 * @category   Oara_Curl
 * @copyright  Fubra Limited
 * @version    Release: 01.00
 *
 */
class Oara_Curl_Access{
	/**
	 * Curl options.
	 * @var array
	 */
	private $_options = array();
	/**
	 * If we are connected to the website.
	 * @var boolean
	 */
	private $_connected = false;
	/**
	 * Number of threads
	 * @var integer
	 */
	private $_threads = 7;

	/**
	 * Constructor and Login.
	 * @param $url - Url Login
	 * @param $valuesLogin - Array with the login parameters
	 * @return none
	 */
	public function __construct($url, array $valuesLogin, $credentials) {

		if (!isset($credentials["cookiesDir"])){
			$credentials["cookiesDir"] = "Oara";
		}
		if (!isset($credentials["cookiesSubDir"])){
			$credentials["cookiesSubDir"] = "Import";
		}
		if (!isset($credentials["cookieName"])){
			$credentials["cookieName"] = "default";
		}


		//Setting cookies
		$isOldSMG = $credentials['networkName'] == "Smg" && (strpos($credentials["cookieName"], 'old') !== false);
		$isTD = ($credentials['networkName'] == "TradeDoubler" || $credentials['networkName'] == "Stream20");
		//$isAW = $credentials['networkName'] == "AffiliateWindow";
		$dir = realpath(dirname(__FILE__)).'/../data/curl/'.$credentials['cookiesDir'].'/'.$credentials['cookiesSubDir'].'/';
			
		if (!Oara_Utilities::mkdir_recursive($dir,0777)){
			throw new Exception ('Problem creating folder in Access');
		}
		//Deleting the last cookie

		if ($handle = opendir($dir)) {
			/* This is the correct way to loop over the directory. */
			while (false !== ($file = readdir($handle))) {
				if ($credentials['cookieName'] == strstr($file, '_', true)){
					unlink($dir.$file);
					break;
				}
			}
			closedir($handle);
		}


		$cookieName = $credentials["cookieName"];

		$cookies = $dir.$cookieName.'_cookies.txt';

		$this->_options = array(
		CURLOPT_USERAGENT => "Mozilla/5.0 (X11; Linux i686; rv:7.0.1) Gecko/20100101 Firefox/7.0.1",
		CURLOPT_RETURNTRANSFER => true,
		CURLOPT_FAILONERROR => true,
		CURLOPT_COOKIEJAR  => $cookies,
		CURLOPT_COOKIEFILE => $cookies,
		CURLOPT_FAILONERROR => true,
		CURLOPT_HTTPAUTH => CURLAUTH_ANY,
		CURLOPT_AUTOREFERER => true,
		CURLOPT_SSL_VERIFYPEER =>false,
		//CURLOPT_VERBOSE => true,
		);

		//Init curl
		$ch = curl_init();
		$options = $this->_options;
		$options[CURLOPT_URL] = $url;
		$options[CURLOPT_POST] = true;

		$options[CURLOPT_FOLLOWLOCATION] = true;

		// Login form fields
		$arg = self::getPostFields($valuesLogin);

		$options[CURLOPT_POSTFIELDS] = $arg;

		//problem with SMG about the redirects and headers
		if ($isOldSMG || $isTD){
			$options[CURLOPT_FOLLOWLOCATION] = false;
			$options[CURLOPT_HEADER] = true;
		}

		curl_setopt_array($ch, $options);

		$result = curl_exec($ch);
		$err = curl_errno($ch);
		$errmsg = curl_error($ch);
		$info = curl_getinfo($ch);



		//Close curl session
		curl_close($ch);

		while (($isOldSMG || $isTD) && ($info['http_code'] == 301 || $info['http_code'] == 302)) { 
			// redirect manually, cookies must be set, which curl does not itself

			// extract new location
			preg_match_all('|Location: (.*)\n|U', $result, $results);
			$location = implode(';', $results[1]);
			$ch = curl_init();

			$options = $this->_options;
			$options[CURLOPT_URL] = str_replace("/publisher/..", "", $location);
			$options[CURLOPT_HEADER] = true;
			$options[CURLOPT_FOLLOWLOCATION] = false;

			curl_setopt_array($ch, $options);

			$result = curl_exec($ch);
			$err = curl_errno($ch);
			$errmsg = curl_error($ch);
			$info = curl_getinfo($ch);

			curl_close($ch);
		}
		echo $result;
		if ($result == false){<commit_msg>	modified:   Oara/Curl/Access.php<commit_after><?php
/**
 * Acces Class
 *
 * @author     <NAME>
 * @category   Oara_Curl
 * @copyright  Fubra Limited
 * @version    Release: 01.00
 *
 */
class Oara_Curl_Access{
	/**
	 * Curl options.
	 * @var array
	 */
	private $_options = array();
	/**
	 * If we are connected to the website.
	 * @var boolean
	 */
	private $_connected = false;
	/**
	 * Number of threads
	 * @var integer
	 */
	private $_threads = 7;

	/**
	 * Constructor and Login.
	 * @param $url - Url Login
	 * @param $valuesLogin - Array with the login parameters
	 * @return none
	 */
	public function __construct($url, array $valuesLogin, $credentials) {

		if (!isset($credentials["cookiesDir"])){
			$credentials["cookiesDir"] = "Oara";
		}
		if (!isset($credentials["cookiesSubDir"])){
			$credentials["cookiesSubDir"] = "Import";
		}
		if (!isset($credentials["cookieName"])){
			$credentials["cookieName"] = "default";
		}


		//Setting cookies
		$isOldSMG = $credentials['networkName'] == "Smg" && (strpos($credentials["cookieName"], 'old') !== false);
		$isTD = ($credentials['networkName'] == "TradeDoubler" || $credentials['networkName'] == "Stream20");
		//$isAW = $credentials['networkName'] == "AffiliateWindow";
		$dir = realpath(dirname(__FILE__)).'/../data/curl/'.$credentials['cookiesDir'].'/'.$credentials['cookiesSubDir'].'/';
			
		if (!Oara_Utilities::mkdir_recursive($dir,0777)){
			throw new Exception ('Problem creating folder in Access');
		}
		//Deleting the last cookie

		if ($handle = opendir($dir)) {
			/* This is the correct way to loop over the directory. */
			while (false !== ($file = readdir($handle))) {
				if ($credentials['cookieName'] == strstr($file, '_', true)){
					unlink($dir.$file);
					break;
				}
			}
			closedir($handle);
		}


		$cookieName = $credentials["cookieName"];

		$cookies = $dir.$cookieName.'_cookies.txt';

		$this->_options = array(
		CURLOPT_USERAGENT => "Mozilla/5.0 (X11; Linux i686; rv:7.0.1) Gecko/20100101 Firefox/7.0.1",
		CURLOPT_RETURNTRANSFER => true,
		CURLOPT_FAILONERROR => true,
		CURLOPT_COOKIEJAR  => $cookies,
		CURLOPT_COOKIEFILE => $cookies,
		CURLOPT_FAILONERROR => true,
		CURLOPT_HTTPAUTH => CURLAUTH_ANY,
		CURLOPT_AUTOREFERER => true,
		CURLOPT_SSL_VERIFYPEER =>false,
		//CURLOPT_VERBOSE => true,
		);

		//Init curl
		$ch = curl_init();
		$options = $this->_options;
		$options[CURLOPT_URL] = $url;
		$options[CURLOPT_POST] = true;

		$options[CURLOPT_FOLLOWLOCATION] = true;

		// Login form fields
		$arg = self::getPostFields($valuesLogin);

		$options[CURLOPT_POSTFIELDS] = $arg;

		//problem with SMG about the redirects and headers
		if ($isOldSMG || $isTD){
			$options[CURLOPT_FOLLOWLOCATION] = false;
			$options[CURLOPT_HEADER] = true;
		}

		curl_setopt_array($ch, $options);

		$result = curl_exec($ch);
		$err = curl_errno($ch);
		$errmsg = curl_error($ch);
		$info = curl_getinfo($ch);



		//Close curl session
		curl_close($ch);

		while (($isOldSMG || $isTD) && ($info['http_code'] == 301 || $info['http_code'] == 302)) { 
			// redirect manually, cookies must be set, which curl does not itself

			// extract new location
			preg_match_all('|Location: (.*)\n|U', $result, $results);
			$location = implode(';', $results[1]);
			$ch = curl_init();

			$options = $this->_options;
			$options[CURLOPT_URL] = str_replace("/publisher/..", "", $location);
			$options[CURLOPT_HEADER] = true;
			$options[CURLOPT_FOLLOWLOCATION] = false;

			curl_setopt_array($ch, $options);

			$result = curl_exec($ch);
			$err = curl_errno($ch);
			$errmsg = curl_error($ch);
			$info = curl_getinfo($ch);

			curl_close($ch);
		}
		//echo $result;
		if ($result == false){
{'fraction_non_alphanumeric': 0.17884405670665213, 'fraction_numerical': 0.010632497273718648, 'mean_word_length': 3.9507422402159245, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4185234', 'n_tokens_mistral': 2982, 'n_tokens_neox': 2601, 'n_words': 535}