<commit_before>import ceylon.buffer.charset {
    utf16
}
import ceylon.buffer.codec {
    DecodeException,
    ignore
}
import ceylon.test {
    assertEquals,
    test,
    assertThatException
}

shared class Utf16Tests() {
    test
    shared void decodeZero() {
        assertEquals {
            utf16.decode({});
            "";
        };
    }
    
    test
    shared void encodeZero() {
        assertEquals {
            utf16.encode("");
            {};
        };
    }
    
    test
    shared void decodeUnicodeLittleEndianBom() {
        assertEquals {
            utf16.decode(
                {
                    #ff, #fe, #bb, #16, #da, #16, #c7, #16, #cf, #16,
                    #aa, #16, #be, #16, #ec, #16
                }*.byte
            );
            "ᚻᛚᛇᛏᚪᚾ᛬";
        };
    }
    
    test
    shared void decodeUnicodeLittleEndianNoBomError() {
        assertThatException(
            () => utf16.decode(
                    {
                        #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa, #16,
                        #be, #16, #ec, #16
                    }*.byte
                )
        )
            .hasType(`DecodeException`)
            .hasMessage("Invalid UTF-16 low surrogate value: 50966");
    }
    
    test
    shared void decodeUnicodeLittleEndianNoBomErrorIgnore() {
        assertEquals {
            utf16.decode(
                {
                    #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa, #16,
                    #be, #16, #ec, #16
                }*.byte,
                ignore
            );
            "묖";
        };
    }
    
    test
    shared void decodeUnicodeBigEndianBom() {
        assertEquals {
            utf16.decode(
                {
                    #fe, #ff, #16, #bb, #16, #da, #16, #c7, #16, #cf,
                    #16, #aa, #16, #be, #16, #ec
                }*.byte
            );
            "ᚻᛚᛇᛏᚪᚾ᛬";
        };
    }
    
    test
    shared void decodeUnicodeBigEndianNoBom() {
        assertEquals {
            utf16.decode(
                {
                    #16, #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa,
                    #16, #be, #16, #ec
                }*.byte
            );
            "ᚻᛚᛇᛏᚪᚾ᛬";
        };
    }
    
    test
    shared void encodeUnicode() {
        assertEquals {
            utf16.encode("ᚻᛚᛇᛏᚪᚾ᛬");
            Array(
                {
                    #16, #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa,
                    #16, #be, #16, #ec
                }*.byte
            );
        };
    }
}<commit_msg>Update utf16.ceylon<commit_after>import ceylon.buffer.charset {
    utf16
}
import ceylon.buffer.codec {
    DecodeException,
    ignore
}
import ceylon.test {
    assertEquals,
    test,
    assertThatException
}

shared class Utf16Tests() {
    test
    shared void decodeZero() {
        assertEquals {
            utf16.decode({});
            "";
        };
    }
    
    test
    shared void encodeZero() {
        assertEquals {
            utf16.encode("");
            {};
        };
    }
    
    test
    shared void decodeUnicodeLittleEndianBom() {
        assertEquals {
            utf16.decode(
                {
                    #ff, #fe, #bb, #16, #da, #16, #c7, #16, #cf, #16,
                    #aa, #16, #be, #16, #ec, #16
                }*.byte
            );
            "ᚻᛚᛇᛏᚪᚾ᛬";
        };
    }
    
    test
    shared void decodeUnicodeLittleEndianNoBomError() {
        assertThatException(
            () => utf16.decode(
                    {
                        #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa, #16,
                        #be, #16, #ec, #16
                    }*.byte
                )
        )
            .hasType(`DecodeException`)
            .hasMessage("Invalid UTF-16 low surrogate value: 50966");
    }
    
    test
    shared void decodeUnicodeLittleEndianNoBomErrorIgnore() {
        assertEquals {
            utf16.decode(
                {
                    #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa, #16,
                    #be, #16, #ec, #16
                }*.byte,
                ignore
            );
            "묖";
        };
    }
    
    test
    shared void decodeUnicodeBigEndianBom() {
        assertEquals {
            utf16.decode(
                {
                    #fe, #ff, #16, #bb, #16, #da, #16, #c7, #16, #cf,
                    #16, #aa, #16, #be, #16, #ec
                }*.byte
            );
            "ᚻᛚᛇᛏᚪᚾ᛬";
        };
    }
    
    test
    shared void decodeUnicodeBigEndianNoBom() {
        assertEquals {
            utf16.decode(
                {
                    #16, #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa,
                    #16, #be, #16, #ec
                }*.byte
            );
            "ᚻᛚᛇᛏᚪᚾ᛬";
        };
    }
    
    test
    shared void encodeUnicode() {
        assertEquals {
            utf16.encode("ᚻᛚᛇᛏᚪᚾ᛬");
            Array(
                {
                    #16, #bb, #16, #da, #16, #c7, #16, #cf, #16, #aa,
                    #16, #be, #16, #ec
                }*.byte
            );
        };
    }
    
    test
    shared void lastCodePoint() {
        assertEquals("\{#10FFFF}", utf16.decode(utf16.encode("\{#10FFFF}")));
    }
}
{'fraction_non_alphanumeric': 0.13795066413662238, 'fraction_numerical': 0.04629981024667932, 'mean_word_length': 1.036707882534776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3516181', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1756, 'n_words': 347}