<commit_before><?php
require_once 'utils.php';

// Stats table ids
define('STATS_DATABASE_STATUS', 1);
define('STATS_LAST_MEMORY_BASE_TABLE_SYNS', 10);
define('STATS_LAST_MEMORY_GEO_TABLE_SYNS', 11);
define('STATS_BASE_ROWN_NUMS', 20);
define('STATS_GEO_ROWN_NUMS', 21);
define('STATS_BASE_DAYSTART_NUM', 22);
define('STATS_GEO_DAYSTART_NUM', 23);

// Database status
define('DATABASE_PREPARE', 0);
define('DATABASE_ACTIVE', 1);

// Query type
define('SQL_UNKNOWN', -1);
define('SQL_BASE_INSERT', 0);
define('SQL_BASE_SELECT', 1);
define('SQL_GEO_INSERT', 2);
define('SQL_GEO_SELECT', 3);
define('SQL_STATS_INSERT', 4);
define('SQL_STATS_SELECT', 5);
define('SQL_GEO_UPDATE', 6);
define('SQL_BASE_UPDATE', 7);

function db_connect()
{
	global $db;
	global $dbUseMemory;

	$result = false;
	$tries = 3;
	while (!$result && $tries--)
	{
		/* Подключаемся к БД */
		$db = mysqli_connect(DB_SERV, DB_USER, DB_PASS, DB_NAME);

		/* Проверка подключения */
		if($db->connect_errno)
		{
			return false;
		}
		else
		{
			$db->set_charset('utf8');
			$result = true;
		}
	}

	$dbUseMemory = false;
	$DataBaseStatus = GetStatsValue(STATS_DATABASE_STATUS);
	if($DataBaseStatus == DATABASE_PREPARE) // Database not avaible (now loading to mem)
	{
		$db = NULL;
		$result = false;
	}

	if(!TRY_USE_MEMORY_TABLES && $DataBaseStatus != DATABASE_ACTIVE)
	{
		SetStatsValue(STATS_DATABASE_STATUS, DATABASE_ACTIVE, true);
	}
	else
	{
		if(TRY_USE_MEMORY_TABLES)
		{
			$dbUseMemory = true;
			/*if ($DataBaseStatus == -1)
			{	// Service table not initialized
				$db = NULL;
				$result = false;
			}*/
		}
	}
	return $result;
}

function GetStatsValue($StatId)
{
	$res = QuerySql('SELECT value FROM STATS_TABLE WHERE StatId='.$StatId);
	if(!$res) return false;
	if($res->num_rows < 1) return -1;

	$row = $res->fetch_row();
	$res->close();
	return $row[0];
}

function SetStatsValue($StatId, $Value, $Replace = false)
{
	if($Replace)
	{
		$res = QuerySql('REPLACE INTO STATS_TABLE SET StatId='.$StatId.', value='.$Value);
	}
	else
	{
		$res = QuerySql('UPDATE STATS_TABLE SET value='.$Value.' WHERE StatId='.$StatId);
	}
	return (bool)$res;
}

function FixSql($sql)
{
	// Easy fixes
	$sql = str_replace("\t", '', $sql);
	$sql = str_replace("\n", '', $sql);
	$sql = str_replace("\r", '', $sql);
	$sql = preg_replace("/( ){2,}/", ' ', $sql);
	$sql = preg_replace("/%{2,}/", '%', $sql); // %%... => %

	// Sql fixes
	$sql = preg_replace("/AND (\"|'|`)[A-z]* LIKE (\"|')%(\"|') /", '', $sql);
	$sql = preg_replace("/(\"|'|`)[A-z]* LIKE (\"|')%(\"|') AND/", '', $sql);

	return $sql;
}
<commit_msg>DB: Do not replace important characters in strings<commit_after><?php
require_once 'utils.php';

// Stats table ids
define('STATS_DATABASE_STATUS', 1);
define('STATS_LAST_MEMORY_BASE_TABLE_SYNS', 10);
define('STATS_LAST_MEMORY_GEO_TABLE_SYNS', 11);
define('STATS_BASE_ROWN_NUMS', 20);
define('STATS_GEO_ROWN_NUMS', 21);
define('STATS_BASE_DAYSTART_NUM', 22);
define('STATS_GEO_DAYSTART_NUM', 23);

// Database status
define('DATABASE_PREPARE', 0);
define('DATABASE_ACTIVE', 1);

// Query type
define('SQL_UNKNOWN', -1);
define('SQL_BASE_INSERT', 0);
define('SQL_BASE_SELECT', 1);
define('SQL_GEO_INSERT', 2);
define('SQL_GEO_SELECT', 3);
define('SQL_STATS_INSERT', 4);
define('SQL_STATS_SELECT', 5);
define('SQL_GEO_UPDATE', 6);
define('SQL_BASE_UPDATE', 7);

function db_connect()
{
	global $db;
	global $dbUseMemory;

	$result = false;
	$tries = 3;
	while (!$result && $tries--)
	{
		/* Подключаемся к БД */
		$db = mysqli_connect(DB_SERV, DB_USER, DB_PASS, DB_NAME);

		/* Проверка подключения */
		if($db->connect_errno)
		{
			return false;
		}
		else
		{
			$db->set_charset('utf8');
			$result = true;
		}
	}

	$dbUseMemory = false;
	$DataBaseStatus = GetStatsValue(STATS_DATABASE_STATUS);
	if($DataBaseStatus == DATABASE_PREPARE) // Database not avaible (now loading to mem)
	{
		$db = NULL;
		$result = false;
	}

	if(!TRY_USE_MEMORY_TABLES && $DataBaseStatus != DATABASE_ACTIVE)
	{
		SetStatsValue(STATS_DATABASE_STATUS, DATABASE_ACTIVE, true);
	}
	else
	{
		if(TRY_USE_MEMORY_TABLES)
		{
			$dbUseMemory = true;
			/*if ($DataBaseStatus == -1)
			{	// Service table not initialized
				$db = NULL;
				$result = false;
			}*/
		}
	}
	return $result;
}

function GetStatsValue($StatId)
{
	$res = QuerySql('SELECT value FROM STATS_TABLE WHERE StatId='.$StatId);
	if(!$res) return false;
	if($res->num_rows < 1) return -1;

	$row = $res->fetch_row();
	$res->close();
	return $row[0];
}

function SetStatsValue($StatId, $Value, $Replace = false)
{
	if($Replace)
	{
		$res = QuerySql('REPLACE INTO STATS_TABLE SET StatId='.$StatId.', value='.$Value);
	}
	else
	{
		$res = QuerySql('UPDATE STATS_TABLE SET value='.$Value.' WHERE StatId='.$StatId);
	}
	return (bool)$res;
}

function FixSql($sql)
{
	// Easy fixes
	$sql = str_replace("\t", '', $sql);
	$sql = str_replace("\n", '', $sql);
	$sql = str_replace("\r", '', $sql);
	//$sql = preg_replace("/( ){2,}/", ' ', $sql);
	//$sql = preg_replace("/%{2,}/", '%', $sql); // %%... => %

	// Sql fixes
	$sql = preg_replace("/AND (\"|'|`)[A-z]* LIKE (\"|')%(\"|') /", '', $sql);
	$sql = preg_replace("/(\"|'|`)[A-z]* LIKE (\"|')%(\"|') AND/", '', $sql);

	return $sql;
}

{'fraction_non_alphanumeric': 0.20600115406809003, 'fraction_numerical': 0.012310059626851317, 'mean_word_length': 4.603448275862069, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6102652', 'n_tokens_mistral': 2251, 'n_tokens_neox': 2096, 'n_words': 395}