<commit_before>package net.pwall.util;

import static org.junit.Assert.*;

import java.io.IOException;

import org.junit.Test;

public class TestStrings {

    @Test
    public void test_fromUTF8_1() {
        for (int i = 0; i < 0xD7FF; i++) {
            StringBuilder sb = new StringBuilder();
            try {
                Strings.appendUTF16(sb, i);
            }
            catch (IOException e) {
                // can't happen
            }
            String str1 = sb.toString();
            byte[] bb = Strings.toUTF8(str1);
            assertEquals(str1, Strings.fromUTF8(bb));
        }
    }

    @Test
    public void test_fromUTF8_2() {
        for (int i = 0xE000; i < 0x10FFFF; i++) {
            StringBuilder sb = new StringBuilder();
            try {
                Strings.appendUTF16(sb, i);
            }
            catch (IOException e) {
                // can't happen
            }
            String str1 = sb.toString();
            byte[] bb = Strings.toUTF8(str1);
            assertEquals(str1, Strings.fromUTF8(bb));
        }
    }

    @Test
    public void test_split_S_S() {
        String s1 = "the quick brown fox jumps over the lazy dog";
        String s2 = "the";
        String[] result = Strings.split(s1, s2);
        assertEquals(3, result.length);
        assertEquals("", result[0]);
        assertEquals(" quick brown fox jumps over ", result[1]);
        assertEquals(" lazy dog", result[2]);
        s2 = "dog";
        result = Strings.split(s1, s2);
        assertEquals(2, result.length);
        assertEquals("the quick brown fox jumps over the lazy ", result[0]);
        assertEquals("", result[1]);
        s2 = " ";
        result = Strings.split(s1, s2);
        assertEquals(9, result.length);
        assertEquals("the", result[0]);
        assertEquals("quick", result[1]);
        assertEquals("brown", result[2]);
        assertEquals("fox", result[3]);
        assertEquals("jumps", result[4]);
        assertEquals("over", result[5]);
        assertEquals("the", result[6]);
        assertEquals("lazy", result[7]);
        assertEquals("dog", result[8]);
        s2 = "*";
        result = Strings.split(s1, s2);
        assertEquals(1, result.length);
        assertEquals(s1, result[0]);
    }<commit_msg>Added JUnit<commit_after>package net.pwall.util;

import static org.junit.Assert.*;

import java.io.IOException;

import org.junit.Test;

public class TestStrings {

    @Test
    public void test_fromUTF8_1() {
        for (int i = 0; i < 0xD7FF; i++) {
            StringBuilder sb = new StringBuilder();
            try {
                Strings.appendUTF16(sb, i);
            }
            catch (IOException e) {
                // can't happen
            }
            String str1 = sb.toString();
            byte[] bb = Strings.toUTF8(str1);
            assertEquals(str1, Strings.fromUTF8(bb));
        }
    }

    @Test
    public void test_fromUTF8_2() {
        for (int i = 0xE000; i < 0x10FFFF; i++) {
            StringBuilder sb = new StringBuilder();
            try {
                Strings.appendUTF16(sb, i);
            }
            catch (IOException e) {
                // can't happen
            }
            String str1 = sb.toString();
            byte[] bb = Strings.toUTF8(str1);
            assertEquals(str1, Strings.fromUTF8(bb));
        }
    }

    @Test
    public void test_split_S_S() {
        String s1 = "the quick brown fox jumps over the lazy dog";
        String s2 = "the";
        String[] result = Strings.split(s1, s2);
        assertEquals(3, result.length);
        assertEquals("", result[0]);
        assertEquals(" quick brown fox jumps over ", result[1]);
        assertEquals(" lazy dog", result[2]);
        s2 = "dog";
        result = Strings.split(s1, s2);
        assertEquals(2, result.length);
        assertEquals("the quick brown fox jumps over the lazy ", result[0]);
        assertEquals("", result[1]);
        s2 = " ";
        result = Strings.split(s1, s2);
        assertEquals(9, result.length);
        assertEquals("the", result[0]);
        assertEquals("quick", result[1]);
        assertEquals("brown", result[2]);
        assertEquals("fox", result[3]);
        assertEquals("jumps", result[4]);
        assertEquals("over", result[5]);
        assertEquals("the", result[6]);
        assertEquals("lazy", result[7]);
        assertEquals("dog", result[8]);
        s2 = "*";
        result = Strings.split(s1, s2);
        assertEquals(1, result.length);
        assertEquals(s1, result[0]);
    }

    @Test
    public void test_split_ST() {
        String s1 = "the quick brown fox jumps over the lazy dog";
        String[] result = Strings.split(s1, ch -> ch == 'o');
        assertEquals(5, result.length);
        assertEquals("the quick br", result[0]);
        assertEquals("wn f", result[1]);
        assertEquals("x jumps ", result[2]);
        assertEquals("ver the lazy d", result[3]);
        assertEquals("g", result[4]);
    }
{'fraction_non_alphanumeric': 0.1380686352753392, 'fraction_numerical': 0.025937749401436554, 'mean_word_length': 1.936731107205624, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4255162', 'n_tokens_mistral': 1472, 'n_tokens_neox': 1412, 'n_words': 363}