<commit_before>public class Strings {
  private static void expect(boolean v) {
    if (! v) throw new RuntimeException();
  }

  private static boolean equal(Object a, Object b) {
    return a == b || (a != null && a.equals(b));
  }

  private static boolean arraysEqual(Object[] a, Object[] b) {
    if (a.length != b.length) {
      return false;
    }

    for (int i = 0; i < a.length; ++i) {
      if (! equal(a[i], b[i])) {
        return false;
      }
    }

    return true;
  }

  private static void testDecode(final boolean prematureEOS) throws Exception {
    java.io.Reader r = new java.io.InputStreamReader
      (new java.io.InputStream() {
          int state = 0;

          public int read() {
            throw new UnsupportedOperationException();
          }

          public int read(byte[] b, int offset, int length) {
            if (length == 0) return 0;

            switch (state) {
            case 0:
              b[offset] = (byte) 0xc2;
              state = 1;
              return 1;

            case 1:
              b[offset] = (byte) 0xae;
              state = 2;
              return 1;

            case 2:
              b[offset] = (byte) 0xea;
              state = 3;
              return 1;

            case 3:
              b[offset] = (byte) 0xba;
              state = prematureEOS ? 5 : 4;
              return 1;

            case 4:
              b[offset] = (byte) 0xaf;
              state = 5;
              return 1;

            case 5:
              return -1;

            default:
              throw new RuntimeException();
            }
          }
        });

    char[] buffer = new char[2];
    int offset = 0;
    while (offset < buffer.length) {<commit_msg>specify UTF-8 explicitly in Strings.testDecode<commit_after>public class Strings {
  private static void expect(boolean v) {
    if (! v) throw new RuntimeException();
  }

  private static boolean equal(Object a, Object b) {
    return a == b || (a != null && a.equals(b));
  }

  private static boolean arraysEqual(Object[] a, Object[] b) {
    if (a.length != b.length) {
      return false;
    }

    for (int i = 0; i < a.length; ++i) {
      if (! equal(a[i], b[i])) {
        return false;
      }
    }

    return true;
  }

  private static void testDecode(final boolean prematureEOS) throws Exception {
    java.io.Reader r = new java.io.InputStreamReader
      (new java.io.InputStream() {
          int state = 0;

          public int read() {
            throw new UnsupportedOperationException();
          }

          public int read(byte[] b, int offset, int length) {
            if (length == 0) return 0;

            switch (state) {
            case 0:
              b[offset] = (byte) 0xc2;
              state = 1;
              return 1;

            case 1:
              b[offset] = (byte) 0xae;
              state = 2;
              return 1;

            case 2:
              b[offset] = (byte) 0xea;
              state = 3;
              return 1;

            case 3:
              b[offset] = (byte) 0xba;
              state = prematureEOS ? 5 : 4;
              return 1;

            case 4:
              b[offset] = (byte) 0xaf;
              state = 5;
              return 1;

            case 5:
              return -1;

            default:
              throw new RuntimeException();
            }
          }
        }, "UTF-8");

    char[] buffer = new char[2];
    int offset = 0;
    while (offset < buffer.length) {
{'fraction_non_alphanumeric': 0.11092052601486564, 'fraction_numerical': 0.017724413950829045, 'mean_word_length': 1.3218314532183146, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5585998', 'n_tokens_mistral': 1076, 'n_tokens_neox': 986, 'n_words': 315}