<commit_before>import utf8 from "../utf-8.es6";

const assert = require("assert"),
    pairs = [
        [0, [0]],
        [0x00007f, [0b01111111]],
        [0x000080, [0b11000010, 0b10000000]],
        [0x0007ff, [0b11011111, 0b10111111]],
        [0x000800, [0b11100000, 0b10100000, 0b10000000]],
        [0x00ffff, [0b11101111, 0b10111111, 0b10111111]],
        [0x010000, [0b11110000, 0b10010000, 0b10000000, 0b10000000]],
        [0x10ffff, [0b11110100, 0b10001111, 0b10111111, 0b10111111]]
    ],
    invalidUtf8Tuples = [
        [
            [0b01111111, 0b10000000]
        ], [
            [0b11000000],
            [0b11000000, 0b00000000],
            [0b11000000, 0b10000000, 0b10000000]
        ], [
            [0b11100000],
            [0b11100000, 0b10000000],
            [0b11100000, 0b10000000, 0b00000000],
            [0b11100000, 0b10000000, 0b10000000, 0b10000000]
        ], [
            [0b11110000],
            [0b11110000, 0b10000000],
            [0b11110000, 0b10000000, 0b10000000],
            [0b11110000, 0b10000000, 0b10000000, 0b00000000],
            [0b11110000, 0b10000000, 0b10000000, 0b10000000, 0b10000000]
        ]
    ];

describe("Code Points", function() {
    describe("#fromCodePoint()", function() {
        for(let pair of pairs) {
            it(`code point ${pair[0].toString(16)} -> UTF-8 ${pair[1].map((ele) => ele.toString(16)).join(" ")}`, function() {
                assert.deepEqual(utf8.fromCodePoint(pair[0]), pair[1]);
            });
        }

        it(`throws if code point lower than 0`, function() {
            assert.throws(function() {
                utf8.fromCodePoint(-1);
            }, RangeError);
        });

        it(`throws if code point higher than 0x10FFFF`, function() {
            assert.throws(function() {
                utf8.fromCodePoint(0x110000);
            }, RangeError);
        });
    });

    describe("#toCodePoint()", function() {
        for(let pair of pairs) {
            it(`UTF-8 ${pair[1].map((ele) => ele.toString(16)).join(" ")} -> code point ${pair[0]}`, function() {
                assert.equal(utf8.toCodePoint(pair[1]), pair[0]);
            });
        }

        it(`throws if not an array`, function() {
            assert.throws(function() {
                utf8.toCodePoint();
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint(0);
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint(null);
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint({});
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint("");
            }, TypeError);
        });

        it(`throws if empty array`, function() {
            assert.throws(function() {
                utf8.toCodePoint([]);
            }, Error);
        });

        it(`throws if array element negative`, function() {
            assert.throws(function() {
                utf8.toCodePoint([-1]);
            }, Error);
        });

        for(let i = 0, length = invalidUtf8Tuples.length; i < length; ++i) {
            for(let tuple of invalidUtf8Tuples[i]) {
                it(`throws if byte count isn't ${i + 1} for ${tuple.map((ele) => ele.toString(16)).join(" ")}`, function() {
                    assert.throws(function() {
                        utf8.toCodePoint(tuple);
                    }, Error);
                });
            }
        }
    });
})
<commit_msg>Add test for utf8.toCodePoint(uint8Array)<commit_after>import utf8 from "../utf-8.es6";

const assert = require("assert"),
    pairs = [
        [0, [0]],
        [0x00007f, [0b01111111]],
        [0x000080, [0b11000010, 0b10000000]],
        [0x0007ff, [0b11011111, 0b10111111]],
        [0x000800, [0b11100000, 0b10100000, 0b10000000]],
        [0x00ffff, [0b11101111, 0b10111111, 0b10111111]],
        [0x010000, [0b11110000, 0b10010000, 0b10000000, 0b10000000]],
        [0x10ffff, [0b11110100, 0b10001111, 0b10111111, 0b10111111]]
    ],
    invalidUtf8Tuples = [
        [
            [0b01111111, 0b10000000]
        ], [
            [0b11000000],
            [0b11000000, 0b00000000],
            [0b11000000, 0b10000000, 0b10000000]
        ], [
            [0b11100000],
            [0b11100000, 0b10000000],
            [0b11100000, 0b10000000, 0b00000000],
            [0b11100000, 0b10000000, 0b10000000, 0b10000000]
        ], [
            [0b11110000],
            [0b11110000, 0b10000000],
            [0b11110000, 0b10000000, 0b10000000],
            [0b11110000, 0b10000000, 0b10000000, 0b00000000],
            [0b11110000, 0b10000000, 0b10000000, 0b10000000, 0b10000000]
        ]
    ];

describe("Code Points", function() {
    describe("#fromCodePoint()", function() {
        for(let pair of pairs) {
            it(`code point ${pair[0].toString(16)} -> UTF-8 ${pair[1].map((ele) => ele.toString(16)).join(" ")}`, function() {
                assert.deepEqual(utf8.fromCodePoint(pair[0]), pair[1]);
            });
        }

        it(`throws if code point lower than 0`, function() {
            assert.throws(function() {
                utf8.fromCodePoint(-1);
            }, RangeError);
        });

        it(`throws if code point higher than 0x10FFFF`, function() {
            assert.throws(function() {
                utf8.fromCodePoint(0x110000);
            }, RangeError);
        });
    });

    describe("#toCodePoint()", function() {
        for(let pair of pairs) {
            it(`UTF-8 ${pair[1].map((ele) => ele.toString(16)).join(" ")} -> code point ${pair[0]}`, function() {
                assert.equal(utf8.toCodePoint(pair[1]), pair[0]);
            });
        }

        it(`works for Array and Uint8Array`, function() {
            for(let pair of pairs) {
                assert.equal(utf8.toCodePoint(new Uint8Array(pair[1])), pair[0]);
            }
        });

        it(`throws if not an array`, function() {
            assert.throws(function() {
                utf8.toCodePoint();
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint(0);
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint(null);
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint({});
            }, TypeError);
            assert.throws(function() {
                utf8.toCodePoint("");
            }, TypeError);
        });

        it(`throws if empty array`, function() {
            assert.throws(function() {
                utf8.toCodePoint([]);
            }, Error);
        });

        it(`throws if array element negative`, function() {
            assert.throws(function() {
                utf8.toCodePoint([-1]);
            }, Error);
        });

        for(let i = 0, length = invalidUtf8Tuples.length; i < length; ++i) {
            for(let tuple of invalidUtf8Tuples[i]) {
                it(`throws if byte count isn't ${i + 1} for ${tuple.map((ele) => ele.toString(16)).join(" ")}`, function() {
                    assert.throws(function() {
                        utf8.toCodePoint(tuple);
                    }, Error);
                });
            }
        }
    });
})

{'fraction_non_alphanumeric': 0.15106441048034935, 'fraction_numerical': 0.15379366812227074, 'mean_word_length': 1.7316436824450243, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 52, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7208600', 'n_tokens_mistral': 3032, 'n_tokens_neox': 2234, 'n_words': 394}