<commit_before><?php

include(OPENFUEGO_DIR . '/openfuego-metadata-functions.php');

function openfuego_echo($string) {
	if (OPENFUEGO_DEBUG) {
    	echo $string;
		return TRUE;
    } else {
    	return FALSE;
	}
}

function openfuego_get_dbh($db_host = OPENFUEGO_DB_HOST, $db_port = OPENFUEGO_DB_PORT, $db_name = OPENFUEGO_DB_NAME, $db_user = OPENFUEGO_DB_USER, $db_pass = <PASSWORD>UEGO_DB_PASS, $driver = OPENFUEGO_DB_DRIVER) {

	$dsn = "$driver:host=$db_host;port=$db_port;dbname=$db_name";

	$try = 1;
	
	while ($try <= 3) {	
		try {
			$dbh = new PDO($dsn, $db_user, $db_pass, array(
				PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES 'utf8' COLLATE 'utf8_unicode_ci';",
				PDO::ATTR_PERSISTENT => true
			));
			$dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
			return $dbh;
		}
		
		catch (PDOException $e) {
			openfuego_notify('OpenFuego exception in ' . __FUNCTION__, $e);
			sleep($try*30);
			$try++;
			continue;
		}
	}
	return FALSE;
}


function openfuego_strpos_arr($haystack, $needles, $before = NULL, $after = NULL) { 
    foreach($needles as $needle) { 
        if(($pos = strpos($haystack, $before . $needle . $after)) !== FALSE) return $pos; 
    }
 return FALSE;
} 


function openfuego_notify($subject, $body = NULL) {
	
	if (OPENFUEGO_DEBUG) echo $subject . "\n" . $body;
	mail(OPENFUEGO_WEBMASTER, $subject, print_r($body, TRUE), 'From: ' . OPENFUEGO_POSTMASTER);
	return TRUE;
}


function array_most_common($input) {
	$counted = array_count_values($input);
	arsort($counted);
	return($counted);
}


function openfuego_get_universe($min_influence = 1) {
	
	try {
		$dbh = openfuego_get_dbh();
		$sth = $dbh->prepare("SELECT user_id FROM openfuego_citizens WHERE influence >= :min_influence;");
		$sth->bindParam('min_influence', $min_influence);
		$sth->execute();
		
		$user_ids = $sth->fetchAll(PDO::FETCH_COLUMN);

		return $user_ids;

	} catch (PDOException $e) {
		openfuego_notify('OpenFuego exception in ' . __FUNCTION__, $e);
		return FALSE;
	}
}


function openfuego_clean_up() {

	$expiration_int = OPENFUEGO_EXPIRATION_INT;
	$now = time();
	$date = date('Y-m-d H:i:s', $now);
	
	$dbh = openfuego_get_dbh();
	
	$sql = "DELETE FROM openfuego_links WHERE first_seen < DATE_SUB(:date, INTERVAL :expiration_int DAY);
			DELETE FROM openfuego_short_links WHERE last_seen < DATE_SUB(:date, INTERVAL :expiration_int DAY);";
	$sth = $dbh->prepare($sql);
	$sth->bindParam('date', $date, PDO::PARAM_INT);
	$sth->bindParam('expiration_int', $expiration_int, PDO::PARAM_INT);
	$sth->execute();

  return TRUE;
 }


function read_body($ch, $chunk) {
  static $range = 5000;
  static $data = '';
/* 		  static $limit = 10000; */

  $len = strlen($data) + strlen($chunk);
  if ($len >= $range) {
    $data .= substr($chunk, 0, $range-strlen($data));
//    print_r($data);
//   echo strlen($data) . ' ' . $data;
//    return -1;
			return $data;
  }

  $data .= $chunk;
  
  return strlen($chunk);
}


function openfuego_curl($url, $method = 'GET', $headers = FALSE, $limit = FALSE) {

	$ch = curl_init($url);

	$options = array(
		CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
		CURLOPT_REFERER => 'http://www.google.com',
		CURLOPT_CONNECTTIMEOUT => 15,
		CURLOPT_TIMEOUT => 15,
		CURLOPT_RETURNTRANSFER => TRUE,<commit_msg>Whoops, fixed useragent and referer<commit_after><?php

include(OPENFUEGO_DIR . '/openfuego-metadata-functions.php');

function openfuego_echo($string) {
	if (OPENFUEGO_DEBUG) {
    	echo $string;
		return TRUE;
    } else {
    	return FALSE;
	}
}

function openfuego_get_dbh($db_host = OPENFUEGO_DB_HOST, $db_port = OPENFUEGO_DB_PORT, $db_name = OPENFUEGO_DB_NAME, $db_user = OPENFUEGO_DB_USER, $db_pass = <PASSWORD>, $driver = OPENFUEGO_DB_DRIVER) {

	$dsn = "$driver:host=$db_host;port=$db_port;dbname=$db_name";

	$try = 1;
	
	while ($try <= 3) {	
		try {
			$dbh = new PDO($dsn, $db_user, $db_pass, array(
				PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES 'utf8' COLLATE 'utf8_unicode_ci';",
				PDO::ATTR_PERSISTENT => true
			));
			$dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
			return $dbh;
		}
		
		catch (PDOException $e) {
			openfuego_notify('OpenFuego exception in ' . __FUNCTION__, $e);
			sleep($try*30);
			$try++;
			continue;
		}
	}
	return FALSE;
}


function openfuego_strpos_arr($haystack, $needles, $before = NULL, $after = NULL) { 
    foreach($needles as $needle) { 
        if(($pos = strpos($haystack, $before . $needle . $after)) !== FALSE) return $pos; 
    }
 return FALSE;
} 


function openfuego_notify($subject, $body = NULL) {
	
	if (OPENFUEGO_DEBUG) echo $subject . "\n" . $body;
	mail(OPENFUEGO_WEBMASTER, $subject, print_r($body, TRUE), 'From: ' . OPENFUEGO_POSTMASTER);
	return TRUE;
}


function array_most_common($input) {
	$counted = array_count_values($input);
	arsort($counted);
	return($counted);
}


function openfuego_get_universe($min_influence = 1) {
	
	try {
		$dbh = openfuego_get_dbh();
		$sth = $dbh->prepare("SELECT user_id FROM openfuego_citizens WHERE influence >= :min_influence;");
		$sth->bindParam('min_influence', $min_influence);
		$sth->execute();
		
		$user_ids = $sth->fetchAll(PDO::FETCH_COLUMN);

		return $user_ids;

	} catch (PDOException $e) {
		openfuego_notify('OpenFuego exception in ' . __FUNCTION__, $e);
		return FALSE;
	}
}


function openfuego_clean_up() {

	$expiration_int = OPENFUEGO_EXPIRATION_INT;
	$now = time();
	$date = date('Y-m-d H:i:s', $now);
	
	$dbh = openfuego_get_dbh();
	
	$sql = "DELETE FROM openfuego_links WHERE first_seen < DATE_SUB(:date, INTERVAL :expiration_int DAY);
			DELETE FROM openfuego_short_links WHERE last_seen < DATE_SUB(:date, INTERVAL :expiration_int DAY);";
	$sth = $dbh->prepare($sql);
	$sth->bindParam('date', $date, PDO::PARAM_INT);
	$sth->bindParam('expiration_int', $expiration_int, PDO::PARAM_INT);
	$sth->execute();

  return TRUE;
 }


function read_body($ch, $chunk) {
  static $range = 5000;
  static $data = '';
/* 		  static $limit = 10000; */

  $len = strlen($data) + strlen($chunk);
  if ($len >= $range) {
    $data .= substr($chunk, 0, $range-strlen($data));
//    print_r($data);
//   echo strlen($data) . ' ' . $data;
//    return -1;
			return $data;
  }

  $data .= $chunk;
  
  return strlen($chunk);
}


function openfuego_curl($url, $method = 'GET', $headers = FALSE, $limit = FALSE) {

	$ch = curl_init($url);

	$options = array(
		CURLOPT_USERAGENT => OPENFUEGO_USER_AGENT,
		CURLOPT_REFERER => OPENFUEGO_REFERER,
		CURLOPT_CONNECTTIMEOUT => 15,
		CURLOPT_TIMEOUT => 15,
		CURLOPT_RETURNTRANSFER => TRUE,
{'fraction_non_alphanumeric': 0.1634205721241631, 'fraction_numerical': 0.007303712720632989, 'mean_word_length': 4.405427631578948, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479297', 'n_tokens_mistral': 2795, 'n_tokens_neox': 2560, 'n_words': 482}