<commit_before><!DOCTYPE html>

<html>
    <head>
        <title>Browser Vp8 Validation Tests</title>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">

    </head>
    <body>
        <pre id="output" >TODO write content</pre>

        <script src="../node_modules/jsivf/builds/jsivf.js"></script>
        <script src="../node_modules/js-md5/build/md5.min.js"></script>
        <script src="../builds/jsvpx.js"></script>
        <script>

            function pad(num, size) {
                var s = num + "";
                while (s.length < size)
                    s = "0" + s;
                return s;
            }

            var vectorFolder = '../vp8-test-vectors/';
            var output = document.getElementById('output');

            var vector = 'vp80-00-comprehensive-001.ivf';
            var testVectorRequest = new XMLHttpRequest();
            testVectorRequest.open("GET", vectorFolder + vector, true);
            testVectorRequest.responseType = "arraybuffer";

            testVectorRequest.onload = function (event) {

                runTest(testVectorRequest.response);

            };

            testVectorRequest.send(null);

            function runTest(buffer) {
                var demuxer = new JsIvf();

                demuxer.receiveBuffer(buffer);
                demuxer.parseHeader();

                var decoder = new JsVpx();
                var outData = "";

                for (var i = 0; i < demuxer.frameCount; i++) {
                    var testImg = decoder.decode(demuxer.processFrame());
                    if (testImg) {
                        var offsets = testImg.planes_off;
                        var y_off = offsets[0];
                        var u_off = offsets[1];
                        var v_off = offsets[2];


                        var mainHash = md5.create();

                        for (var y = 0; y < testImg.d_h; y++) {
                            mainHash.update(testImg.img_data.subarray(y_off, y_off + testImg.d_w));
                            y_off += testImg.stride[0];
                        }

                        var chromaHeight = ((1 + testImg.d_h) / 2) | 0;
                        var chromaWidth = ((1 + testImg.d_w) / 2) | 0;


                        //UPDATING MAIN HASH U
                        u_off = offsets[1];
                        for (var y = 0; y < chromaHeight; y++) //testImg.d_h
                        {


                            mainHash.update(testImg.img_data.subarray(u_off, u_off + chromaWidth));
                            u_off += testImg.stride[1];
                        }

                        //UPDATING MAIN HASH V
                        v_off = offsets[2];
                        for (var y = 0; y < chromaHeight; y++) //testImg.d_h
                        {

                            mainHash.update(testImg.img_data.subarray(v_off, v_off + chromaWidth));
                            v_off += testImg.stride[2];
                        }


                        outData += mainHash.hex() + "  " + 'basename here' + "-";
                        outData += testImg.d_w + "x" + testImg.d_h;
                        outData += "-" + pad(i, 4);
                        outData += ".i420";
                        outData += "\n";

                    }
                    output.innerHTML = outData;
                }

            }

            function validateTest() {

            }
<commit_msg>Got a simple qunit test working<commit_after><!DOCTYPE html>

<html>
    <head>
        <title>Browser Vp8 Validation Tests</title>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <link rel="stylesheet" href="https://code.jquery.com/qunit/qunit-2.1.1.css">
    </head>
    <body>
        <div id="qunit"></div>
        <div id="qunit-fixture"></div>
        <script src="https://code.jquery.com/qunit/qunit-2.1.1.js"></script>
        <script src="../node_modules/jsivf/builds/jsivf.js"></script>
        <script src="../node_modules/js-md5/build/md5.min.js"></script>
        <script src="../builds/jsvpx.js"></script>
        <script>

            function pad(num, size) {
                var s = num + "";
                while (s.length < size)
                    s = "0" + s;
                return s;
            }

            var vectorFolder = '../vp8-test-vectors/';
            var output = document.getElementById('output');

            var vector = 'vp80-00-comprehensive-001.ivf';
            var testVectorRequest = new XMLHttpRequest();
            testVectorRequest.open("GET", vectorFolder + vector, true);
            testVectorRequest.responseType = "arraybuffer";

            testVectorRequest.onload = function (event) {

                runTest(testVectorRequest.response, vector);

            };

            testVectorRequest.send(null);

            function runTest(buffer, vector) {
                var demuxer = new JsIvf();

                demuxer.receiveBuffer(buffer);
                demuxer.parseHeader();

                var decoder = new JsVpx();
                var outData = "";

                for (var i = 0; i < demuxer.frameCount; i++) {
                    var testImg = decoder.decode(demuxer.processFrame());
                    if (testImg) {
                        var offsets = testImg.planes_off;
                        var y_off = offsets[0];
                        var u_off = offsets[1];
                        var v_off = offsets[2];


                        var mainHash = md5.create();

                        for (var y = 0; y < testImg.d_h; y++) {
                            mainHash.update(testImg.img_data.subarray(y_off, y_off + testImg.d_w));
                            y_off += testImg.stride[0];
                        }

                        var chromaHeight = ((1 + testImg.d_h) / 2) | 0;
                        var chromaWidth = ((1 + testImg.d_w) / 2) | 0;


                        //UPDATING MAIN HASH U
                        u_off = offsets[1];
                        for (var y = 0; y < chromaHeight; y++) //testImg.d_h
                        {


                            mainHash.update(testImg.img_data.subarray(u_off, u_off + chromaWidth));
                            u_off += testImg.stride[1];
                        }

                        //UPDATING MAIN HASH V
                        v_off = offsets[2];
                        for (var y = 0; y < chromaHeight; y++) //testImg.d_h
                        {

                            mainHash.update(testImg.img_data.subarray(v_off, v_off + chromaWidth));
                            v_off += testImg.stride[2];
                        }


                        outData += mainHash.hex() + "  " + vector.split('.')[0] + "-";
                        outData += testImg.d_w + "x" + testImg.d_h;
                        outData += "-" + pad((i + 1), 4);
                        outData += ".i420";
                        outData += "\n";

                    }


                }
                var validationRecordRequest = new XMLHttpRequest();
                validationRecordRequest.open("GET", vectorFolder + vector + '.md5', true);
                validationRecordRequest.responseType = "text";

                validationRecordRequest.onload = function (event) {
                    //output.innerHTML = outData;
                    var validData = validationRecordRequest.response;
                    console.warn(outData);
                    console.warn(validData);
                    QUnit.test(vector, function (assert) {
                        assert.strictEqual(outData , validData, "Md5 Vector Comparison");
                    });
                };

                validationRecordRequest.send(null);


            }

    


{'fraction_non_alphanumeric': 0.11926489226869455, 'fraction_numerical': 0.010899873257287706, 'mean_word_length': 1.163696188648204, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 44, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1017452', 'n_tokens_mistral': 2271, 'n_tokens_neox': 2052, 'n_words': 413}