<commit_before>package mclint.patterns;

public class UnparsedPattern {
  private String originalPattern;
  private String pattern;

  public static UnparsedPattern fromString(String pattern) {
    return new UnparsedPattern(pattern);
  }

  private UnparsedPattern(String pattern) {
    this.originalPattern = pattern;
    this.pattern = pattern.replaceAll("\\s+", "");
  }

  public boolean consume(String token) {
    if (pattern.startsWith(token)) {
      pattern = pattern.substring(token.length());
      return true;
    }
    return false;
  }

  public boolean startsWithMeta() {
    return pattern.length() >= 2 && pattern.charAt(0) == '%'
        && Character.isLetter(pattern.charAt(1));
  }
  
  public boolean emptyAfterMeta() {
    return pattern.substring(2).isEmpty();
  }
  
  // Precondition: startsWithMeta()
  public char popMeta() {
    char meta = pattern.charAt(1);
    pattern = pattern.substring(2);
    return meta;
  }
  
  public String asString() {
    return originalPattern;
  }
  
  @Override
  public String toString() {
    return String.format("<UnparsedPattern: %s>", originalPattern);
  }
}<commit_msg>More efficient implementation of UnparsedPattern, using indices instead of substrings<commit_after>package mclint.patterns;

public class UnparsedPattern {
  private String pattern;
  private int index = 0;

  public static UnparsedPattern fromString(String pattern) {
    return new UnparsedPattern(pattern);
  }

  private UnparsedPattern(String pattern) {
    this.pattern = pattern;
  }
  
  private void advance() {
    if (index >= pattern.length()) {
      return;
    }
    do {
      ++index;
    } while (index < pattern.length() && Character.isWhitespace(pattern.charAt(index)));
  }
  
  private void advance(int by) {
    for (int i = 0; i < by; ++i) {
      advance();
    }
  }

  public boolean consume(String token) {
    if (pattern.regionMatches(index, token, 0, token.length())) {
      advance(token.length());
      return true;
    }
    return false;
  }

  public boolean startsWithMeta() {
    return pattern.length() - index >= 2 && pattern.charAt(index) == '%'
        && Character.isLetter(pattern.charAt(index + 1));
  }
  
  public boolean emptyAfterMeta() {
    return index + 2 >= pattern.length();
  }
  
  // Precondition: startsWithMeta()
  public char popMeta() {
    char meta = pattern.charAt(index + 1);
    advance(2);
    return meta;
  }
  
  public String asString() {
    return pattern;
  }
  
  @Override
  public String toString() {
    return String.format("<UnparsedPattern: %s>", pattern);
  }
}
{'fraction_non_alphanumeric': 0.12374323279195669, 'fraction_numerical': 0.005413766434648105, 'mean_word_length': 3.17258064516129, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '11623', 'n_tokens_mistral': 783, 'n_tokens_neox': 781, 'n_words': 191}