<commit_before><?php

function curl_from_json($url, $array=false, $decode=true) {

	$ch = curl_init();
    curl_setopt($ch, CURLOPT_USERAGENT,'Data.gov data.json crawler');
	curl_setopt($ch, CURLOPT_URL, $url);
	
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
	curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);	
	
    curl_setopt($ch, CURLOPT_FRESH_CONNECT, true);
    curl_setopt($ch, CURLOPT_FILETIME, true);
    
    curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET'); 
    
    curl_setopt($ch, CURLOPT_COOKIESESSION, true);
    curl_setopt($ch, CURLOPT_COOKIE, "");
    
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);    
    

	$data=curl_exec($ch);
	curl_close($ch);

    if($decode == true) {
      $data = iconv('UTF-8', 'UTF-8//IGNORE', utf8_encode($data));
	    return json_decode($data, $array);	        
    } else {
        return $data;
    }


}


function curl_header($url) {
	$info = array();
	
	$ch = curl_init();
//	curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 5);
    curl_setopt($ch, CURLOPT_USERAGENT,'Data.gov data.json crawler');
    curl_setopt($ch, CURLOPT_URL, $url);

	curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
	curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);	
	
    curl_setopt($ch, CURLOPT_FRESH_CONNECT, true);
    curl_setopt($ch, CURLOPT_FILETIME, true);
    
    curl_setopt($ch, CURLOPT_NOBODY, true);
    
    curl_setopt($ch, CURLOPT_COOKIESESSION, true);
    curl_setopt($ch, CURLOPT_COOKIE, "");
    
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);

    $info['header'] = curl_exec($ch);
    $info['info'] = curl_getinfo($ch);
    curl_close($ch);

	return $info;
}

/**
 * This mashes together two arrays with the same keys
 * It fills in any empty values, but gives precedence to the $primary array
 */

function array_mash($primary, $secondary) {
	$primary = (array)$primary;
	$secondary = (array)$secondary;<commit_msg>Set CURL timeout<commit_after><?php

function curl_from_json($url, $array=false, $decode=true) {

	$ch = curl_init();
    curl_setopt($ch, CURLOPT_USERAGENT,'Data.gov data.json crawler');
	curl_setopt($ch, CURLOPT_URL, $url);
	
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
	curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);	
	
    curl_setopt($ch, CURLOPT_FRESH_CONNECT, true);
    curl_setopt($ch, CURLOPT_FILETIME, true);
    
    curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET'); 
    
    curl_setopt($ch, CURLOPT_COOKIESESSION, true);
    curl_setopt($ch, CURLOPT_COOKIE, "");
    
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);    
    

	$data=curl_exec($ch);
	curl_close($ch);

    if($decode == true) {
      $data = iconv('UTF-8', 'UTF-8//IGNORE', utf8_encode($data));
	    return json_decode($data, $array);	        
    } else {
        return $data;
    }


}


function curl_header($url) {
	$info = array();
	
	$ch = curl_init();
//	curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 5);
    curl_setopt($ch, CURLOPT_USERAGENT,'Data.gov data.json crawler');
    curl_setopt($ch, CURLOPT_URL, $url);

	curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
	curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);	

    curl_setopt($ch, CURLOPT_TIMEOUT, 10);  
	
    curl_setopt($ch, CURLOPT_FRESH_CONNECT, true);
    curl_setopt($ch, CURLOPT_FILETIME, true);
    
    curl_setopt($ch, CURLOPT_NOBODY, true);
    
    curl_setopt($ch, CURLOPT_COOKIESESSION, true);
    curl_setopt($ch, CURLOPT_COOKIE, "");
    
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);

    $info['header'] = curl_exec($ch);
    $info['info'] = curl_getinfo($ch);
    curl_close($ch);

	return $info;
}

/**
 * This mashes together two arrays with the same keys
 * It fills in any empty values, but gives precedence to the $primary array
 */

function array_mash($primary, $secondary) {
	$primary = (array)$primary;
	$secondary = (array)$secondary;
{'fraction_non_alphanumeric': 0.14794243877808635, 'fraction_numerical': 0.00555415299166877, 'mean_word_length': 4.303882195448461, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3084393', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1465, 'n_words': 268}