<commit_before><?php

chdir(__DIR__);

$startTime = time();

require_once('../incl/incl.php');
require_once('../incl/heartbeat.incl.php');

RunMeNTimes(1);
CatchKill();

ini_set('memory_limit','256M');

if (!DBConnect())
    DebugMessage('Cannot connect to db!', E_USER_ERROR);

define('ITEM_COL_AVG', 0);
define('ITEM_COL_DAYS', 1);
define('ITEM_COL_MIN', 2);
define('ITEM_COL_MAX', 3);

heartbeat();
file_put_contents('../addon/MarketData-US.lua', BuildAddonData('US'));
file_put_contents('../addon/MarketData-EU.lua', BuildAddonData('EU'));
MakeZip();

DebugMessage('Done! Started '.TimeDiff($startTime));

function BuildAddonData($region)
{
    global $db, $caughtKill;

    heartbeat();
    if ($caughtKill)
        return;

    DebugMessage("Starting region $region");;

    $globalSpots = 3; // number of global prices in front of every string

    $stmt = $db->prepare('select distinct house from tblRealm where region = ? and canonical is not null order by 1');
    $stmt->bind_param('s', $region);
    $stmt->execute();
    $result = $stmt->get_result();
    $houses = DBMapArray($result, null);
    $stmt->close();

    $items = [];
    $ihmColsTemplate = '';
    for ($x = 1; $x <= 31; $x++) {
        $ihmColsTemplate .= " when $x then ihm%1\$s.mktslvr".($x < 10 ? '0' : '')."$x";
    }
    $ihmCols = sprintf($ihmColsTemplate, '');
    $ihm2Cols = sprintf($ihmColsTemplate, '2');
    $ihm3Cols = sprintf($ihmColsTemplate, '3');

    $sql = <<<EOF
SELECT tis.item,
ifnull(i.stacksize,0) stacksize,
datediff(now(), tis.lastseen) since,
round(tis.price/100) lastprice,
ifnull(ihd.priceavg, case day(hc.lastdaily) $ihmCols end) priced1,
ifnull(ihd2.priceavg, case day(timestampadd(day, -1, hc.lastdaily)) $ihm2Cols end) priced2,
ifnull(ihd3.priceavg, case day(timestampadd(day, -2, hc.lastdaily)) $ihm3Cols end) priced3,
least(ihd.pricemin, ihd2.pricemin, ihd3.pricemin) pricemin,
greatest(ihd.pricemax, ihd2.pricemax, ihd3.pricemax) pricemax
FROM tblItemSummary tis
join tblHouseCheck hc on hc.house = tis.house
join tblDBCItem i on tis.item=i.id
left join tblItemHistoryDaily ihd on ihd.house=tis.house and ihd.item=tis.item and ihd.`when` = hc.lastdaily
left join tblItemHistoryDaily ihd2 on ihd2.house=tis.house and ihd2.item=tis.item and ihd2.`when` = timestampadd(day, -1, hc.lastdaily)
left join tblItemHistoryDaily ihd3 on ihd3.house=tis.house and ihd3.item=tis.item and ihd3.`when` = timestampadd(day, -2, hc.lastdaily)
left join tblItemHistoryMonthly ihm on ihm.house=tis.house and ihm.item=tis.item and ihm.month=((year(hc.lastdaily) - 2014) * 12 + month(hc.lastdaily))
left join tblItemHistoryMonthly ihm2 on ihm2.house=tis.house and ihm2.item=tis.item and ihm2.month=((year(timestampadd(day, -1, hc.lastdaily)) - 2014) * 12 + month(timestampadd(day, -1, hc.lastdaily)))
left join tblItemHistoryMonthly ihm3 on ihm3.house=tis.house and ihm3.item=tis.item and ihm3.month=((year(timestampadd(day, -2, hc.lastdaily)) - 2014) * 12 + month(timestampadd(day, -2, hc.lastdaily)))
WHERE tis.house = ?
EOF;

    for ($hx = 0; $hx < count($houses); $hx++) {
        heartbeat();
        if ($caughtKill)
            return;

        DebugMessage('Finding prices in house '.$houses[$hx].' ('.round($hx/count($houses)*100).'%) '.round(memory_get_usage()/1048576));

        $stmt = $db->prepare($sql);
        $stmt->bind_param('i', $houses[$hx]);
        $stmt->execute();
        $result = $stmt->get_result();
        while ($priceRow = $result->fetch_assoc()) {
            $item = intval($priceRow['item'],10);
            $avg = intval($priceRow['lastprice'],0);

            $dailyPrices = [$priceRow['priced1'], $priceRow['priced2'], $priceRow['priced3']];
            for ($x = 0; $x < count($dailyPrices); $x++) {
                if (is_null($dailyPrices[$x])) {
                    array_splice($dailyPrices, $x--, 1);
                }
            }
            if (count($dailyPrices) > 0) {
                $avg = round(array_sum($dailyPrices)/count($dailyPrices));
            }

            $items[$item][$hx+$globalSpots] = [
                ITEM_COL_AVG => $avg,<commit_msg>do not use array when calculating daily price avg<commit_after><?php

chdir(__DIR__);

$startTime = time();

require_once('../incl/incl.php');
require_once('../incl/heartbeat.incl.php');

RunMeNTimes(1);
CatchKill();

ini_set('memory_limit','256M');

if (!DBConnect())
    DebugMessage('Cannot connect to db!', E_USER_ERROR);

define('ITEM_COL_AVG', 0);
define('ITEM_COL_DAYS', 1);
define('ITEM_COL_MIN', 2);
define('ITEM_COL_MAX', 3);

heartbeat();
file_put_contents('../addon/MarketData-US.lua', BuildAddonData('US'));
file_put_contents('../addon/MarketData-EU.lua', BuildAddonData('EU'));
MakeZip();

DebugMessage('Done! Started '.TimeDiff($startTime));

function BuildAddonData($region)
{
    global $db, $caughtKill;

    heartbeat();
    if ($caughtKill)
        return;

    DebugMessage("Starting region $region");;

    $globalSpots = 3; // number of global prices in front of every string

    $stmt = $db->prepare('select distinct house from tblRealm where region = ? and canonical is not null order by 1');
    $stmt->bind_param('s', $region);
    $stmt->execute();
    $result = $stmt->get_result();
    $houses = DBMapArray($result, null);
    $stmt->close();

    $items = [];
    $ihmColsTemplate = '';
    for ($x = 1; $x <= 31; $x++) {
        $ihmColsTemplate .= " when $x then ihm%1\$s.mktslvr".($x < 10 ? '0' : '')."$x";
    }
    $ihmCols = sprintf($ihmColsTemplate, '');
    $ihm2Cols = sprintf($ihmColsTemplate, '2');
    $ihm3Cols = sprintf($ihmColsTemplate, '3');

    $sql = <<<EOF
SELECT tis.item,
ifnull(i.stacksize,0) stacksize,
datediff(now(), tis.lastseen) since,
round(tis.price/100) lastprice,
ifnull(ihd.priceavg, case day(hc.lastdaily) $ihmCols end) priced1,
ifnull(ihd2.priceavg, case day(timestampadd(day, -1, hc.lastdaily)) $ihm2Cols end) priced2,
ifnull(ihd3.priceavg, case day(timestampadd(day, -2, hc.lastdaily)) $ihm3Cols end) priced3,
least(ihd.pricemin, ihd2.pricemin, ihd3.pricemin) pricemin,
greatest(ihd.pricemax, ihd2.pricemax, ihd3.pricemax) pricemax
FROM tblItemSummary tis
join tblHouseCheck hc on hc.house = tis.house
join tblDBCItem i on tis.item=i.id
left join tblItemHistoryDaily ihd on ihd.house=tis.house and ihd.item=tis.item and ihd.`when` = hc.lastdaily
left join tblItemHistoryDaily ihd2 on ihd2.house=tis.house and ihd2.item=tis.item and ihd2.`when` = timestampadd(day, -1, hc.lastdaily)
left join tblItemHistoryDaily ihd3 on ihd3.house=tis.house and ihd3.item=tis.item and ihd3.`when` = timestampadd(day, -2, hc.lastdaily)
left join tblItemHistoryMonthly ihm on ihm.house=tis.house and ihm.item=tis.item and ihm.month=((year(hc.lastdaily) - 2014) * 12 + month(hc.lastdaily))
left join tblItemHistoryMonthly ihm2 on ihm2.house=tis.house and ihm2.item=tis.item and ihm2.month=((year(timestampadd(day, -1, hc.lastdaily)) - 2014) * 12 + month(timestampadd(day, -1, hc.lastdaily)))
left join tblItemHistoryMonthly ihm3 on ihm3.house=tis.house and ihm3.item=tis.item and ihm3.month=((year(timestampadd(day, -2, hc.lastdaily)) - 2014) * 12 + month(timestampadd(day, -2, hc.lastdaily)))
WHERE tis.house = ?
EOF;

    for ($hx = 0; $hx < count($houses); $hx++) {
        heartbeat();
        if ($caughtKill)
            return;

        DebugMessage('Finding prices in house '.$houses[$hx].' ('.round($hx/count($houses)*100).'%) '.round(memory_get_usage()/1048576));

        $stmt = $db->prepare($sql);
        $stmt->bind_param('i', $houses[$hx]);
        $stmt->execute();
        $result = $stmt->get_result();
        while ($priceRow = $result->fetch_assoc()) {
            $item = intval($priceRow['item'],10);

            $c = 0; $s = 0;
            for ($x = 1; $x <=3 ; $x++) {
                if (!is_null($priceRow["priced$x"])) {
                    $s += intval($priceRow["priced$x"], 0);
                    $c++;
                }
            }
            if ($c > 0) {
                $avg = round($s / $c);
            } else {
                $avg = intval($priceRow['lastprice'],0);
            }

            $items[$item][$hx+$globalSpots] = [
                ITEM_COL_AVG => $avg,
{'fraction_non_alphanumeric': 0.17334313365184367, 'fraction_numerical': 0.024010780350361388, 'mean_word_length': 3.5839416058394162, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1321352', 'n_tokens_mistral': 3195, 'n_tokens_neox': 2918, 'n_words': 612}