<commit_before>/* Copyright (c) 2008-2009, <NAME>

   Permission to use, copy, modify, and/or distribute this software
   for any purpose with or without fee is hereby granted, provided
   that the above copyright notice and this permission notice appear
   in all copies.

   There is NO WARRANTY for this software.  See license.txt for
   details. */

package java.lang;

import java.io.UnsupportedEncodingException;
import java.util.regex.Pattern;
import java.util.Comparator;
import java.util.Locale;
import java.io.ByteArrayOutputStream;
import java.io.Serializable;

public final class String
  implements Comparable<String>, CharSequence, Serializable
{
  public static Comparator<String> CASE_INSENSITIVE_ORDER
    = new Comparator<String>() {
    public int compare(String a, String b) {
      return a.compareToIgnoreCase(b);
    }
  };

  private final Object data;
  private final int offset;
  private final int length;
  private int hashCode;

  public String() {
    this(new char[0], 0, 0);
  }

  public String(char[] data, int offset, int length, boolean copy) {
    this((Object) data, offset, length, copy);
  }

  public String(char[] data, int offset, int length) {
    this(data, offset, length, true);
  }

  public String(char[] data) {
    this(data, 0, data.length);
  }

  public String(byte bytes[], int offset, int length, String charsetName)
    throws UnsupportedEncodingException
  {
    this(bytes, offset, length);
    if (! charsetName.equalsIgnoreCase("UTF-8")) {
      throw new UnsupportedEncodingException(charsetName);
    }
  }
  
  public String(byte[] data, int offset, int length, boolean copy) {
    this((Object) data, offset, length, copy);
  }

  public String(byte[] data, int offset, int length) {
    this(data, offset, length, true);
  }

  public String(byte[] data) {
    this(data, 0, data.length);
  }

  public String(String s) {
    this(s.toCharArray());
  }

  public String(byte[] data, String charset)
    throws UnsupportedEncodingException
  {
    this(data);
    if (! charset.equals("UTF-8")) {
      throw new UnsupportedEncodingException(charset);
    }
  }

  public String(byte bytes[], int highByte, int offset, int length) {
    if (offset < 0 || offset + length > bytes.length) {
      throw new IndexOutOfBoundsException
        (offset + " < 0 or " + offset + " + " + length + " > " + bytes.length);
    }

    char[] c = new char[length];
    int mask = highByte << 8;
    for (int i = 0; i < length; ++i) {
      c[i] = (char) ((bytes[offset + i] & 0xFF) | mask);
    }

    this.data = c;
    this.offset = 0;
    this.length = length;
  }

  private String(Object data, int offset, int length, boolean copy) {
    int l;
    if (data instanceof char[]) {
      l = ((char[]) data).length;
    } else {
      l = ((byte[]) data).length;
    }

    if (offset < 0 || offset + length > l) {
      throw new IndexOutOfBoundsException
        (offset + " < 0 or " + offset + " + " + length + " > " + l);
    }

    if(!copy && isUTF8(data)) copy = true;

    if (copy) {
      Object c;
      if (data instanceof char[]) {
        c = new char[length];
        System.arraycopy(data, offset, c, 0, length);
      } else {
        c = decodeUTF8((byte[])data, offset, length);
        if(c instanceof char[]) length = ((char[])c).length;
      }
      
      this.data = c;
      this.offset = 0;
      this.length = length;
    } else {
      this.data = data;
      this.offset = offset;
      this.length = length;
    }
  }

  private static boolean isUTF8(Object data) {
    if(!(data instanceof byte[])) return false;
    byte[] b = (byte[])data;
    for(int i = 0; i < b.length; ++i) {
      if(((int)b[i] & 0x080) != 0) return true;
    }
    return false;
  }

  private static byte[] encodeUTF8(char[] s16, int offset, int length) {
    ByteArrayOutputStream buf = new ByteArrayOutputStream();
    for(int i = offset; i < offset+length; ++i) {
      char c = s16[i];
      if(c == '\u0000') {     // null char
        buf.write(0);
        buf.write(0);
      } else if(c < 0x080) {  // 1 byte char
        buf.write(c);
      } else if(c < 0x0800) { // 2 byte char
        buf.write(0x0c0 | (c >>> 6));
        buf.write(0x080 | (c & 0x03f));
      } else {                // 3 byte char
        buf.write(0x0e0 | ((c >>> 12) & 0x0f));
        buf.write(0x080 | ((c >>> 6) & 0x03f));
        buf.write(0x080 | (c & 0x03f));
      }
    }
    return buf.toByteArray();
  }

  private static void decodeUTF8_insert(Object data, int index, int val) {
    if(data instanceof byte[]) ((byte[])data)[index] = (byte)val;
    else                       ((char[])data)[index] = (char)val;
  }

  private static Object decodeUTF8_widen(Object data, int length, int capacity) {
    byte[] src = (byte[])data;
    char[] result = new char[capacity];
    for(int i = 0; i < length; ++i) result[i] = (char)((int)src[i] & 0x0ff);
    return result;
  }

  private static Object decodeUTF8_trim(Object data, int length) {
    if(data instanceof byte[]) return data;
    if(((char[])data).length == length) return data;
    char[] result = new char[length];
    System.arraycopy(data, 0, result, 0, length);
    return result;
  }

  private static Object decodeUTF8(byte[] s8, int offset, int length) {
    Object buf = new byte[s8.length];
    boolean isMultiByte = false;
    int i=offset, j=0;
    while(i < offset+length) {
      int x = s8[i++];
      if((x & 0x080) == 0x0) {          // 1 byte char
        if(x == 0) ++i;                 // 2 byte null char
        decodeUTF8_insert(buf, j++, x);
      } else if((x & 0x0e0) == 0x0c0) { // 2 byte char
        if(!isMultiByte) {
          buf = decodeUTF8_widen(buf, j, s8.length-1);
          isMultiByte = true;
        }
        int y = s8[i++];
        decodeUTF8_insert(buf, j++, ((x & 0x1f) << 6) | (y & 0x3f));
      } else if((x & 0x0f0) == 0x0e0) { // 3 byte char
        if(!isMultiByte) {
          buf = decodeUTF8_widen(buf, j, s8.length-2);
          isMultiByte = true;
        }
        int y = s8[i++]; int z = s8[i++];
        decodeUTF8_insert(buf, j++, ((x & 0xf) << 12) | ((y & 0x3f) << 6) | (z & 0x3f));
      }
    }

    return decodeUTF8_trim(buf, j);
  }

  public String toString() {
    return this;
  }

  public int length() {
    return length;
  }

  public int hashCode() {
    if (hashCode == 0) {
      int h = 0;
      for (int i = 0; i < length; ++i) h = (h * 31) + charAt(i);
      hashCode = h;
    }
    return hashCode;
  }

  public boolean equals(Object o) {
    return this == o || (o instanceof String && compareTo((String) o) == 0);
  }

  public boolean equalsIgnoreCase(String s) {
    return this == s || compareToIgnoreCase(s) == 0;
  }

  public int compareTo(String s) {
    if (this == s) return 0;

    int idx = 0;
    int result;

    int end = (length < s.length ? length : s.length);
<commit_msg>avoid NPE in String.equalsIgnoreCase<commit_after>/* Copyright (c) 2008-2009, Avian Contributors

   Permission to use, copy, modify, and/or distribute this software
   for any purpose with or without fee is hereby granted, provided
   that the above copyright notice and this permission notice appear
   in all copies.

   There is NO WARRANTY for this software.  See license.txt for
   details. */

package java.lang;

import java.io.UnsupportedEncodingException;
import java.util.regex.Pattern;
import java.util.Comparator;
import java.util.Locale;
import java.io.ByteArrayOutputStream;
import java.io.Serializable;

public final class String
  implements Comparable<String>, CharSequence, Serializable
{
  public static Comparator<String> CASE_INSENSITIVE_ORDER
    = new Comparator<String>() {
    public int compare(String a, String b) {
      return a.compareToIgnoreCase(b);
    }
  };

  private final Object data;
  private final int offset;
  private final int length;
  private int hashCode;

  public String() {
    this(new char[0], 0, 0);
  }

  public String(char[] data, int offset, int length, boolean copy) {
    this((Object) data, offset, length, copy);
  }

  public String(char[] data, int offset, int length) {
    this(data, offset, length, true);
  }

  public String(char[] data) {
    this(data, 0, data.length);
  }

  public String(byte bytes[], int offset, int length, String charsetName)
    throws UnsupportedEncodingException
  {
    this(bytes, offset, length);
    if (! charsetName.equalsIgnoreCase("UTF-8")) {
      throw new UnsupportedEncodingException(charsetName);
    }
  }
  
  public String(byte[] data, int offset, int length, boolean copy) {
    this((Object) data, offset, length, copy);
  }

  public String(byte[] data, int offset, int length) {
    this(data, offset, length, true);
  }

  public String(byte[] data) {
    this(data, 0, data.length);
  }

  public String(String s) {
    this(s.toCharArray());
  }

  public String(byte[] data, String charset)
    throws UnsupportedEncodingException
  {
    this(data);
    if (! charset.equals("UTF-8")) {
      throw new UnsupportedEncodingException(charset);
    }
  }

  public String(byte bytes[], int highByte, int offset, int length) {
    if (offset < 0 || offset + length > bytes.length) {
      throw new IndexOutOfBoundsException
        (offset + " < 0 or " + offset + " + " + length + " > " + bytes.length);
    }

    char[] c = new char[length];
    int mask = highByte << 8;
    for (int i = 0; i < length; ++i) {
      c[i] = (char) ((bytes[offset + i] & 0xFF) | mask);
    }

    this.data = c;
    this.offset = 0;
    this.length = length;
  }

  private String(Object data, int offset, int length, boolean copy) {
    int l;
    if (data instanceof char[]) {
      l = ((char[]) data).length;
    } else {
      l = ((byte[]) data).length;
    }

    if (offset < 0 || offset + length > l) {
      throw new IndexOutOfBoundsException
        (offset + " < 0 or " + offset + " + " + length + " > " + l);
    }

    if(!copy && isUTF8(data)) copy = true;

    if (copy) {
      Object c;
      if (data instanceof char[]) {
        c = new char[length];
        System.arraycopy(data, offset, c, 0, length);
      } else {
        c = decodeUTF8((byte[])data, offset, length);
        if(c instanceof char[]) length = ((char[])c).length;
      }
      
      this.data = c;
      this.offset = 0;
      this.length = length;
    } else {
      this.data = data;
      this.offset = offset;
      this.length = length;
    }
  }

  private static boolean isUTF8(Object data) {
    if(!(data instanceof byte[])) return false;
    byte[] b = (byte[])data;
    for(int i = 0; i < b.length; ++i) {
      if(((int)b[i] & 0x080) != 0) return true;
    }
    return false;
  }

  private static byte[] encodeUTF8(char[] s16, int offset, int length) {
    ByteArrayOutputStream buf = new ByteArrayOutputStream();
    for(int i = offset; i < offset+length; ++i) {
      char c = s16[i];
      if(c == '\u0000') {     // null char
        buf.write(0);
        buf.write(0);
      } else if(c < 0x080) {  // 1 byte char
        buf.write(c);
      } else if(c < 0x0800) { // 2 byte char
        buf.write(0x0c0 | (c >>> 6));
        buf.write(0x080 | (c & 0x03f));
      } else {                // 3 byte char
        buf.write(0x0e0 | ((c >>> 12) & 0x0f));
        buf.write(0x080 | ((c >>> 6) & 0x03f));
        buf.write(0x080 | (c & 0x03f));
      }
    }
    return buf.toByteArray();
  }

  private static void decodeUTF8_insert(Object data, int index, int val) {
    if(data instanceof byte[]) ((byte[])data)[index] = (byte)val;
    else                       ((char[])data)[index] = (char)val;
  }

  private static Object decodeUTF8_widen(Object data, int length, int capacity) {
    byte[] src = (byte[])data;
    char[] result = new char[capacity];
    for(int i = 0; i < length; ++i) result[i] = (char)((int)src[i] & 0x0ff);
    return result;
  }

  private static Object decodeUTF8_trim(Object data, int length) {
    if(data instanceof byte[]) return data;
    if(((char[])data).length == length) return data;
    char[] result = new char[length];
    System.arraycopy(data, 0, result, 0, length);
    return result;
  }

  private static Object decodeUTF8(byte[] s8, int offset, int length) {
    Object buf = new byte[s8.length];
    boolean isMultiByte = false;
    int i=offset, j=0;
    while(i < offset+length) {
      int x = s8[i++];
      if((x & 0x080) == 0x0) {          // 1 byte char
        if(x == 0) ++i;                 // 2 byte null char
        decodeUTF8_insert(buf, j++, x);
      } else if((x & 0x0e0) == 0x0c0) { // 2 byte char
        if(!isMultiByte) {
          buf = decodeUTF8_widen(buf, j, s8.length-1);
          isMultiByte = true;
        }
        int y = s8[i++];
        decodeUTF8_insert(buf, j++, ((x & 0x1f) << 6) | (y & 0x3f));
      } else if((x & 0x0f0) == 0x0e0) { // 3 byte char
        if(!isMultiByte) {
          buf = decodeUTF8_widen(buf, j, s8.length-2);
          isMultiByte = true;
        }
        int y = s8[i++]; int z = s8[i++];
        decodeUTF8_insert(buf, j++, ((x & 0xf) << 12) | ((y & 0x3f) << 6) | (z & 0x3f));
      }
    }

    return decodeUTF8_trim(buf, j);
  }

  public String toString() {
    return this;
  }

  public int length() {
    return length;
  }

  public int hashCode() {
    if (hashCode == 0) {
      int h = 0;
      for (int i = 0; i < length; ++i) h = (h * 31) + charAt(i);
      hashCode = h;
    }
    return hashCode;
  }

  public boolean equals(Object o) {
    return this == o || (o instanceof String && compareTo((String) o) == 0);
  }

  public boolean equalsIgnoreCase(String s) {
    return this == s || (s != null && compareToIgnoreCase(s) == 0);
  }

  public int compareTo(String s) {
    if (this == s) return 0;

    int idx = 0;
    int result;

    int end = (length < s.length ? length : s.length);

{'fraction_non_alphanumeric': 0.145432170262053, 'fraction_numerical': 0.02330968582597365, 'mean_word_length': 2.4728506787330318, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5978165', 'n_tokens_mistral': 4818, 'n_tokens_neox': 4550, 'n_words': 1344}