<commit_before><!DOCTYPE html>
<head xmlns="http://www.w3.org/1999/html" xmlns="http://www.w3.org/1999/html">
    <meta charset="utf-8">

    <script src="vendor/d3/d3.v3.js"></script>

    <link href="vendor/bootstrap/css/bootstrap.css" rel="stylesheet" type="text/css">

    <link rel="stylesheet" href="vendor/highlightjs/7.3/styles/idea.min.css">
    <script src="vendor/highlightjs/7.3/highlight.min.js"></script>

    <style type="text/css">
        pre code {
            white-space: pre;
            white-space: pre-wrap;
        }

        .node circle {
            cursor: pointer;
            fill: #fff;
            stroke: steelblue;
            stroke-width: 1.5px;
        }

        .node text {
            font-size: 12px;
        }

        path.link {
            fill: none;
            stroke: #ccc;
            stroke-width: 1.5px;
        }

        #tasks td:nth-child(4),
        #tasks td:nth-child(5),
        #tasks td:nth-child(6),
        #tasks td:nth-child(7),
        #tasks td:nth-child(8),
        #tasks td:nth-child(9),
        #tasks td:nth-child(10),
        #tasks td:nth-child(11),
        #tasks td:nth-child(12) {
            text-align: right;
        }
    </style>
</head>

<body>


<div class="container">
    <div class="page-header">
        <h1>Presto :: Query <span id="queryId"></span></h1>
    </div>

    <h2>Summary</h2>

    <div class="sql"><pre><code id="query"></code></pre></div>

    <dl class="dl-horizontal">
        <dt>Query State</dt>
        <dd id="queryState"></dd>

        <dt>User</dt>
        <dd id="sessionUser"></dd>

        <dt>Source</dt>
        <dd id="sessionSource"></dd>

        <dt>Catalog</dt>
        <dd id="sessionCatalog"></dd>

        <dt>Schema</dt>
        <dd id="sessionSchema"></dd>

        <dt>Elapsed</dt>
        <dd id="elapsedTime"></dd>

        <dt>Memory</dt>
        <dd id="memory"></dd>

        <dt>CPU Time</dt>
        <dd id="cpuTime"></dd>

        <dt>Rows</dt>
        <dd id="rows"></dd>

        <dt>Data Size</dt>
        <dd id="dataSize"></dd>

        <dt>Raw</dt>
        <dd><a id="rawJson"></a></dd>

        <dt>Message</dt>
        <dd id="failureMessage"></dd>

    </dl>

    <h2>Stages</h2>

    <div id="stages"></div>

    <h2>Tasks</h2>

    <table id="tasks" class="table table-striped">
        <thead>
        <tr>
            <th>Id</th>
            <th>Host</th>
            <th>State</th>
            <th>Rows</th>
            <th>Rows/s</th>
            <th>Bytes</th>
            <th>Bytes/s</th>
            <th>Splits Pending</th>
            <th>Splits Running</th>
            <th>Splits Done</th>
            <th>Output Buffers</th>
            <th>Buffered Pages</th>
        </tr>
        </thead>
    </table>
</div>

</body>

<script>


d3.json('/v1/query/' + window.location.search.substring(1), function (query) {
    d3.select('#query').text(query.query);
    d3.select('#queryId').text(query.queryId);
    d3.select('#queryState').text(query.state);
    d3.select('#sessionUser').text(query.session.user);
    d3.select('#sessionSource').text(query.session.source);
    d3.select('#sessionCatalog').text(query.session.catalog);
    d3.select('#sessionSchema').text(query.session.schema);
    d3.select('#elapsedTime').text(query.queryStats.elapsedTime);
    d3.select('#memory').text(query.queryStats.totalMemoryReservation);
    d3.select('#cpuTime').text(query.queryStats.totalCpuTime);
    d3.select('#rows').text(formatCount(query.queryStats.rawInputPositions));
    d3.select('#dataSize').text(query.queryStats.rawInputDataSize);

    if (query.failureInfo) {
        d3.select('#failureMessage').text(query.failureInfo.type + ": " + query.failureInfo.message);
    }


    var uri = "/v1/query/" + query.queryId + "?pretty"
    d3.select('#rawJson').text(uri);
    d3.select('#rawJson').attr("href", uri);

    hljs.initHighlighting();

    renderStagesTree(query.outputStage);

    renderTable('tasks', getTasks(query.outputStage).sort(taskIdComparator));
});

function removeQueryId(id)
{
    var pos = id.indexOf('.');
    if (pos != -1) {
        return id.substring(pos + 1);
    }

    return id
}

function taskIdComparator(a, b) {
    a = a[0].split('.').map(parseFloat);
    b = b[0].split('.').map(parseFloat);
    return numericArrayComparator(a, b);
}

function numericArrayComparator(a, b) {
    for (var i = 0; (i < a.length) && (i < b.length); i++) {
        if (a[i] < b[i]) return -1;
        if (a[i] > b[i]) return 1;
    }
    if (a.length < b.length) return -1;
    if (a.length > b.length) return 1;
    return 0;
}

function getTasks(stage) {
    return [].concat.apply(
            stage.tasks.map(taskData),
            stage.subStages.map(getTasks));
}

function taskData(d) {
    var elapsed = parseDuration(d.stats.elapsedTime);
    var rows = d.stats.rawInputPositions;
    var bytes = parseDataSize(d.stats.rawInputDataSize);

    var allBuffers = d.outputBuffers.buffers.length;
    var finishedBuffers = d3.sum(d.outputBuffers.buffers, function (i) {
        return i.finished;
    });
    var bufferedPages = d3.sum(d.outputBuffers.buffers, function (i) {
        return i.bufferedPages;
    });

    return [
        d.taskId,
        d.self.replace(/http:\/\/([a-z0-9-.]+)[\/:].*/, '$1'),
        d.state,
        formatCount(rows),
        formatCount(computeRate(rows, elapsed)),
        formatDataSize(bytes),
        formatDataSize(computeRate(bytes, elapsed)),
        d.stats.queuedDrivers,
        d.stats.runningDrivers,
        d.stats.completedDrivers,
        finishedBuffers + ' / ' + allBuffers,
        bufferedPages
    ];
}

function computeRate(count, ms) {
    if (ms == 0) {
        return 0;
    }
    return Math.round(count / ms) / 1000;
}

function formatCount(count) {
    var unit = "";
    if (count > 1000) {
        count /= 1000;
        unit = "K";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "M";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "B";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "T";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "Q";
    }
    return precisionRound(count) + unit;
}


function parseDataSize(value) {
    var DATA_SIZE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$/
    var match = DATA_SIZE_PATTERN.exec(value);
    if (match == null) {
        return null;
    }
    var number = parseInt(match[1]);
    switch (match[2]) {
        case "B":
            return number;
        case "kB":
            return number << 10;
        case "MB":
            return number << 20;
        case "GB":
            return number << 30;
        case "TB":
            return number << 40;
        case "PB":
            return number << 50;
        default:
            return null;
    }
}

function formatDataSize(size) {
    var unit = "B";
    if (size >= 1024) {
        size /= 1024;
        unit = "K";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "M";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "G";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "T";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "P";
    }
    return precisionRound(size) + unit;
}

function parseDuration(value) {
    var DURATION_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$/

    var match = DURATION_PATTERN.exec(value);
    if (match == null) {
        return null;
    }
    var number = parseInt(match[1]);
    switch (match[2]) {
        case "ns":
            return number / 1000000.0;
        case "us":
            return number / 1000.0;;
        case "ms":
            return number;
        case "s":
            return number * 1000;
        case "m":
            return number * 1000 * 60;
        case "h":
            return number * 1000 * 60 * 60;
        case "d":
            return number * 1000 * 60 * 60 * 24;
        default:<commit_msg>Fix over truncation and flow bugs parsing data size and duration<commit_after><!DOCTYPE html>
<head xmlns="http://www.w3.org/1999/html" xmlns="http://www.w3.org/1999/html">
    <meta charset="utf-8">

    <script src="vendor/d3/d3.v3.js"></script>

    <link href="vendor/bootstrap/css/bootstrap.css" rel="stylesheet" type="text/css">

    <link rel="stylesheet" href="vendor/highlightjs/7.3/styles/idea.min.css">
    <script src="vendor/highlightjs/7.3/highlight.min.js"></script>

    <style type="text/css">
        pre code {
            white-space: pre;
            white-space: pre-wrap;
        }

        .node circle {
            cursor: pointer;
            fill: #fff;
            stroke: steelblue;
            stroke-width: 1.5px;
        }

        .node text {
            font-size: 12px;
        }

        path.link {
            fill: none;
            stroke: #ccc;
            stroke-width: 1.5px;
        }

        #tasks td:nth-child(4),
        #tasks td:nth-child(5),
        #tasks td:nth-child(6),
        #tasks td:nth-child(7),
        #tasks td:nth-child(8),
        #tasks td:nth-child(9),
        #tasks td:nth-child(10),
        #tasks td:nth-child(11),
        #tasks td:nth-child(12) {
            text-align: right;
        }
    </style>
</head>

<body>


<div class="container">
    <div class="page-header">
        <h1>Presto :: Query <span id="queryId"></span></h1>
    </div>

    <h2>Summary</h2>

    <div class="sql"><pre><code id="query"></code></pre></div>

    <dl class="dl-horizontal">
        <dt>Query State</dt>
        <dd id="queryState"></dd>

        <dt>User</dt>
        <dd id="sessionUser"></dd>

        <dt>Source</dt>
        <dd id="sessionSource"></dd>

        <dt>Catalog</dt>
        <dd id="sessionCatalog"></dd>

        <dt>Schema</dt>
        <dd id="sessionSchema"></dd>

        <dt>Elapsed</dt>
        <dd id="elapsedTime"></dd>

        <dt>Memory</dt>
        <dd id="memory"></dd>

        <dt>CPU Time</dt>
        <dd id="cpuTime"></dd>

        <dt>Rows</dt>
        <dd id="rows"></dd>

        <dt>Data Size</dt>
        <dd id="dataSize"></dd>

        <dt>Raw</dt>
        <dd><a id="rawJson"></a></dd>

        <dt>Message</dt>
        <dd id="failureMessage"></dd>

    </dl>

    <h2>Stages</h2>

    <div id="stages"></div>

    <h2>Tasks</h2>

    <table id="tasks" class="table table-striped">
        <thead>
        <tr>
            <th>Id</th>
            <th>Host</th>
            <th>State</th>
            <th>Rows</th>
            <th>Rows/s</th>
            <th>Bytes</th>
            <th>Bytes/s</th>
            <th>Splits Pending</th>
            <th>Splits Running</th>
            <th>Splits Done</th>
            <th>Output Buffers</th>
            <th>Buffered Pages</th>
        </tr>
        </thead>
    </table>
</div>

</body>

<script>


d3.json('/v1/query/' + window.location.search.substring(1), function (query) {
    d3.select('#query').text(query.query);
    d3.select('#queryId').text(query.queryId);
    d3.select('#queryState').text(query.state);
    d3.select('#sessionUser').text(query.session.user);
    d3.select('#sessionSource').text(query.session.source);
    d3.select('#sessionCatalog').text(query.session.catalog);
    d3.select('#sessionSchema').text(query.session.schema);
    d3.select('#elapsedTime').text(query.queryStats.elapsedTime);
    d3.select('#memory').text(query.queryStats.totalMemoryReservation);
    d3.select('#cpuTime').text(query.queryStats.totalCpuTime);
    d3.select('#rows').text(formatCount(query.queryStats.rawInputPositions));
    d3.select('#dataSize').text(query.queryStats.rawInputDataSize);

    if (query.failureInfo) {
        d3.select('#failureMessage').text(query.failureInfo.type + ": " + query.failureInfo.message);
    }


    var uri = "/v1/query/" + query.queryId + "?pretty"
    d3.select('#rawJson').text(uri);
    d3.select('#rawJson').attr("href", uri);

    hljs.initHighlighting();

    renderStagesTree(query.outputStage);

    renderTable('tasks', getTasks(query.outputStage).sort(taskIdComparator));
});

function removeQueryId(id)
{
    var pos = id.indexOf('.');
    if (pos != -1) {
        return id.substring(pos + 1);
    }

    return id
}

function taskIdComparator(a, b) {
    a = a[0].split('.').map(parseFloat);
    b = b[0].split('.').map(parseFloat);
    return numericArrayComparator(a, b);
}

function numericArrayComparator(a, b) {
    for (var i = 0; (i < a.length) && (i < b.length); i++) {
        if (a[i] < b[i]) return -1;
        if (a[i] > b[i]) return 1;
    }
    if (a.length < b.length) return -1;
    if (a.length > b.length) return 1;
    return 0;
}

function getTasks(stage) {
    return [].concat.apply(
            stage.tasks.map(taskData),
            stage.subStages.map(getTasks));
}

function taskData(d) {
    var elapsed = parseDuration(d.stats.elapsedTime);
    var rows = d.stats.rawInputPositions;
    var bytes = parseDataSize(d.stats.rawInputDataSize);

    var allBuffers = d.outputBuffers.buffers.length;
    var finishedBuffers = d3.sum(d.outputBuffers.buffers, function (i) {
        return i.finished;
    });
    var bufferedPages = d3.sum(d.outputBuffers.buffers, function (i) {
        return i.bufferedPages;
    });

    return [
        d.taskId,
        d.self.replace(/http:\/\/([a-z0-9-.]+)[\/:].*/, '$1'),
        d.state,
        formatCount(rows),
        formatCount(computeRate(rows, elapsed)),
        formatDataSize(bytes),
        formatDataSize(computeRate(bytes, elapsed)),
        d.stats.queuedDrivers,
        d.stats.runningDrivers,
        d.stats.completedDrivers,
        finishedBuffers + ' / ' + allBuffers,
        bufferedPages
    ];
}

function computeRate(count, ms) {
    if (ms == 0) {
        return 0;
    }
    return Math.round(count / ms) / 1000;
}

function formatCount(count) {
    var unit = "";
    if (count > 1000) {
        count /= 1000;
        unit = "K";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "M";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "B";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "T";
    }
    if (count > 1000) {
        count /= 1000;
        unit = "Q";
    }
    return precisionRound(count) + unit;
}


function parseDataSize(value) {
    var DATA_SIZE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$/
    var match = DATA_SIZE_PATTERN.exec(value);
    if (match == null) {
        return null;
    }
    var number = parseFloat(match[1]);
    switch (match[2]) {
        case "B":
            return number;
        case "kB":
            return number * Math.pow(2, 10);
        case "MB":
            return number * Math.pow(2, 20);
        case "GB":
            return number * Math.pow(2, 30);
        case "TB":
            return number * Math.pow(2, 40);
        case "PB":
            return number * Math.pow(2, 50);
        default:
            return null;
    }
}

function formatDataSize(size) {
    var unit = "B";
    if (size >= 1024) {
        size /= 1024;
        unit = "K";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "M";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "G";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "T";
    }
    if (size >= 1024) {
        size /= 1024;
        unit = "P";
    }
    return precisionRound(size) + unit;
}

function parseDuration(value) {
    var DURATION_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$/

    var match = DURATION_PATTERN.exec(value);
    if (match == null) {
        return null;
    }
    var number = parseFloat(match[1]);
    switch (match[2]) {
        case "ns":
            return number / 1000000.0;
        case "us":
            return number / 1000.0;
        case "ms":
            return number;
        case "s":
            return number * 1000;
        case "m":
            return number * 1000 * 60;
        case "h":
            return number * 1000 * 60 * 60;
        case "d":
            return number * 1000 * 60 * 60 * 24;
        default:
{'fraction_non_alphanumeric': 0.17548226509023024, 'fraction_numerical': 0.027566894835096452, 'mean_word_length': 1.9450247388675097, 'pattern_counts': {'":': 28, '<': 259, '<?xml version=': 0, '>': 265, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2396750', 'n_tokens_mistral': 5885, 'n_tokens_neox': 5432, 'n_words': 1022}