<commit_before>package com.mikesamuel.cil.ptree;

import com.google.common.base.Optional;
import com.mikesamuel.cil.parser.MatchErrorReceiver;
import com.mikesamuel.cil.parser.MatchState;
import com.mikesamuel.cil.parser.ParSer;
import com.mikesamuel.cil.parser.ParSerable;
import com.mikesamuel.cil.parser.ParseErrorReceiver;
import com.mikesamuel.cil.parser.ParseState;
import com.mikesamuel.cil.parser.SerialErrorReceiver;
import com.mikesamuel.cil.parser.SerialState;

final class Completer extends ParSer {
  final ParSerable p;

  Completer(ParSerable p) {
    this.p = p;
  }

  @Override
  public Optional<ParseState> parse(
      ParseState state, ParseErrorReceiver err) {
    Optional<ParseState> next = p.getParSer().parse(state, err);
    if (next.isPresent()) {
      // Test empty except for ignorable tokens.
      ParseState nextState = next.get().advance(0, true);
      if (nextState.isEmpty()) {
        return Optional.of(nextState);
      }
      err.error(state, "Unparsed input");
    }
    return Optional.absent();
  }

  @Override
  public Optional<SerialState> unparse(
      SerialState state, SerialErrorReceiver err) {
    return p.getParSer().unparse(state, err);
  }

  @Override
  public Optional<MatchState> match(
      MatchState state, MatchErrorReceiver r) {
    Optional<MatchState> next = p.getParSer().match(state, r);
    if (next.isPresent()) {
      if (next.get().isEmpty()) {
        return next;
      }
      r.error(state, "Unmatched input");<commit_msg>better error message on incomplete parse<commit_after>package com.mikesamuel.cil.ptree;

import com.google.common.base.Optional;
import com.mikesamuel.cil.parser.MatchErrorReceiver;
import com.mikesamuel.cil.parser.MatchState;
import com.mikesamuel.cil.parser.ParSer;
import com.mikesamuel.cil.parser.ParSerable;
import com.mikesamuel.cil.parser.ParseErrorReceiver;
import com.mikesamuel.cil.parser.ParseState;
import com.mikesamuel.cil.parser.SerialErrorReceiver;
import com.mikesamuel.cil.parser.SerialState;

final class Completer extends ParSer {
  final ParSerable p;

  Completer(ParSerable p) {
    this.p = p;
  }

  @Override
  public Optional<ParseState> parse(
      ParseState state, ParseErrorReceiver err) {
    Optional<ParseState> next = p.getParSer().parse(state, err);
    if (next.isPresent()) {
      // Test empty except for ignorable tokens.
      ParseState nextState = next.get().advance(0, true);
      if (nextState.isEmpty()) {
        return Optional.of(nextState);
      }
      String unparsed =
          nextState.input.content.substring(nextState.indexAfterIgnorables());
      if (unparsed.length() > 10) {
        unparsed = unparsed.substring(0, 10) + "...";
      }
      err.error(nextState, "Unparsed input `" + unparsed + "`");
    }
    return Optional.absent();
  }

  @Override
  public Optional<SerialState> unparse(
      SerialState state, SerialErrorReceiver err) {
    return p.getParSer().unparse(state, err);
  }

  @Override
  public Optional<MatchState> match(
      MatchState state, MatchErrorReceiver r) {
    Optional<MatchState> next = p.getParSer().match(state, r);
    if (next.isPresent()) {
      if (next.get().isEmpty()) {
        return next;
      }
      r.error(state, "Unmatched input");
{'fraction_non_alphanumeric': 0.12089598036207426, 'fraction_numerical': 0.0021478981282602025, 'mean_word_length': 3.6045197740112993, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5007709', 'n_tokens_mistral': 1047, 'n_tokens_neox': 1038, 'n_words': 208}