<commit_before><?

$local_file = "fr.txt";
$country_id = "1";
//$absPath_to_OreonOconf = $path = $oreon->optGen["oreon_web_path"]."oreon.conf.php";
$absPath_to_OreonOconf = "/usr/local/oreon/www/oreon.conf.php";

function microtime_float() {
  list($usec, $sec) = explode(" ", microtime());
  return ((float)$usec + (float)$sec);
}

$time_start = microtime_float();
set_time_limit(3000);

$log_file = fopen($local_file, "r");

require_once('DB.php');
require_once($absPath_to_OreonOconf );

$dsn = array(
	     'phptype'  => 'mysql',
	     'username' => $conf_oreon['user'],
	     'password' => $conf_oreon['password'],
	     'hostspec' => $conf_oreon['host'],
	     'database' => $conf_oreon['db'],
	     );

$options = array(
		 'debug'       => 2,
		 'portability' => DB_PORTABILITY_ALL ^ DB_PORTABILITY_LOWERCASE,
		 );

$db =& DB::connect($dsn, $options);
if (PEAR::isError($db)) {
  die($db->getMessage());
}

$db->setFetchMode(DB_FETCHMODE_ASSOC);

$insert = false;
$current = array();
$cmp = array();
if ($log_file)	{
	while ($str = fgets($log_file))	{
	    $tab = split("\t", $str);
	   /* foreach ($tab as $key=>$value)	{
	    	print "key :".$key." value:".$value."\n";
	    }*/
	    if ($tab[23] != "FULL_NAME_ND")	{
			$date = explode("-", $tab[24]);
			$date = mktime(0, 0, 0, $date[1], $date[2], $date[0]);
			//print "INSERT INTO `view_city` ( `city_id` , `country_id` , `city_name` , `city_zipcode` , `city_lat` , `city_long`, `city_date`) VALUES ('', '".$country_id."', '".$tab[23]."', NULL , '".$tab[3]."', '".$tab[4]."', '".$date."')\n";
			$db->query("INSERT INTO `view_city` ( `city_id` , `country_id` , `city_name` , `city_zipcode` , `city_lat` , `city_long`, `city_date`) VALUES ('', '".$country_id."', '".$tab[23]."', NULL , '".$tab[3]."', '".$tab[4]."', '".$date."')");
		}
	}
}
$time_end = microtime_float();
$now = $time_end - $time_start;
print round($now,3) . "secondes";

?><commit_msg>Get all the city name case, just set a number to multiple version<commit_after><?

$local_file = "";
$country_id = "";
$absPath_to_OreonOconf = "../../../../../oreon.conf.php";

function microtime_float() {
  list($usec, $sec) = explode(" ", microtime());
  return ((float)$usec + (float)$sec);
}

$time_start = microtime_float();
set_time_limit(3000);

$log_file = fopen($local_file, "r");

require_once('DB.php');
require_once($absPath_to_OreonOconf );

$dsn = array(
	     'phptype'  => 'mysql',
	     'username' => $conf_oreon['user'],
	     'password' => $conf_oreon['password'],
	     'hostspec' => $conf_oreon['host'],
	     'database' => $conf_oreon['db'],
	     );

$options = array(
		 'debug'       => 2,
		 'portability' => DB_PORTABILITY_ALL ^ DB_PORTABILITY_LOWERCASE,
		 );

$db =& DB::connect($dsn, $options);
if (PEAR::isError($db)) {
  die($db->getMessage());
}

$db->setFetchMode(DB_FETCHMODE_ASSOC);

$tab = array();
$last_elem = NULL;
$i = 2;
if ($log_file)	{
	for ($cpt = 0; $str = fgets($log_file); $cpt++)	{
	   	$tab = split("\t", $str);
	    if ($cpt && $tab[9] == "P")	{
	    	$lat = $tab[3];
	    	$long = $tab[4];
	    	$tab = array_reverse($tab);			
			$date = explode("-", $tab[0]);
			$date = mktime(0, 0, 0, $date[1], $date[2], $date[0]);
			if ($tab[1] == $last_elem)	{
				$tab[1] = $tab[1]." - ".$i;
				$i++;
			}
			else	{
				$last_elem = $tab[1];
				$i = 2;
			}
			$db->query("INSERT INTO `view_city` ( `city_id` , `country_id` , `city_name` , `city_zipcode` , `city_lat` , `city_long`, `city_date`) VALUES ('', '".$country_id."', '".$tab[1]."', NULL , '".$lat."', '".$long."', '".$date."')");
		}
		unset($str);
	}
}
$time_end = microtime_float();
$now = $time_end - $time_start;
print round($now,3) . "secondes";
?>
{'fraction_non_alphanumeric': 0.25578231292517006, 'fraction_numerical': 0.013333333333333334, 'mean_word_length': 3.954177897574124, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4670913', 'n_tokens_mistral': 1614, 'n_tokens_neox': 1506, 'n_words': 266}