<commit_before><?php
ini_set('memory_limit','1024M');
require_once dirname(__FILE__) . '/../../bootstrap.php';
Core::setUser(UserAccount::get(UserAccount::ID_SYSTEM_ACCOUNT));
echo "Begin at MELB TIME: " . UDate::now(UDate::TIME_ZONE_MELB) . "\n";
$productIds = Dao::getResultsNative('SELECT `id` FROM `product` where active = 1 order by `id` desc');
$productIds = array_map(create_function('$a', 'return intval($a["id"]);'), $productIds);
//$started = array();
//$started['time'] = UDate::now();
//$started['count'] = Record::countByCriteria('active = 1');
echo "Got [ " . count($productIds) . " ] products . " . "\n";
$count = 1;
$sku = '';
foreach ($productIds as $productId)
{
	$product = Product::get($productId);
	$sku = '';
	if ($product instanceof Product)
	{<commit_msg>added truncating data for price match<commit_after><?php
ini_set('memory_limit','1024M');
require_once dirname(__FILE__) . '/../../bootstrap.php';
Core::setUser(UserAccount::get(UserAccount::ID_SYSTEM_ACCOUNT));
echo "Begin at MELB TIME: " . UDate::now(UDate::TIME_ZONE_MELB) . "\n";
$productIds = Dao::getResultsNative('SELECT `id` FROM `product` where active = 1 order by `id` desc');
$productIds = array_map(create_function('$a', 'return intval($a["id"]);'), $productIds);

echo "    starting to archive old data : " . UDate::now(UDate::TIME_ZONE_MELB) . "\n";
$result = Dao::getResultsNative('Call PreparePriceMatch()');
echo "    Finished archiving old data : " . UDate::now(UDate::TIME_ZONE_MELB) . "\n";
//$started = array();
//$started['time'] = UDate::now();
//$started['count'] = Record::countByCriteria('active = 1');
echo "Got [ " . count($productIds) . " ] products . " . "\n";
$count = 1;
$sku = '';
foreach ($productIds as $productId)
{
	$product = Product::get($productId);
	$sku = '';
	if ($product instanceof Product)
	{
{'fraction_non_alphanumeric': 0.21518987341772153, 'fraction_numerical': 0.007705008255365988, 'mean_word_length': 5.7835820895522385, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2261068', 'n_tokens_mistral': 647, 'n_tokens_neox': 628, 'n_words': 135}