<commit_before>package water.api;

import com.google.gson.JsonObject;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Set;
import java.util.regex.Pattern;
import water.*;
import water.parser.CsvParser;
import water.parser.CustomParser;
import water.parser.GuessSetup;
import water.util.RString;

abstract public class Parse extends Request {

  private   final ParserType     _parserType= new ParserType(PARSER_TYPE);
  private   final Separator      _separator = new Separator(SEPARATOR);
  private   final Bool           _header    = new Bool(HEADER,false,"Use first line as a header");
  protected final Bool           _sQuotes   = new Bool("single_quotes",false,"Enable single quotes as field quotation character");
  protected final HeaderKey      _hdrFrom   = new HeaderKey("header_from_file",false);
  protected final Str            _excludeExpression    = new Str("exclude","");
  protected final ExistingCSVKey _source    = new ExistingCSVKey(SOURCE_KEY);
  protected final NewH2OHexKey   _dest      = new NewH2OHexKey(DEST_KEY);
  protected final Bool           _blocking  = new Bool("blocking",false,"Synchronously wait until parse completes");
  @SuppressWarnings("unused")
  private   final Preview        _preview   = new Preview(PREVIEW);

  public Parse() {
    _excludeExpression.setRefreshOnChange();
    _header.setRefreshOnChange();
    _blocking._hideInQuery = true;
  }

//  private static String toHTML(ParseSetupGuessException e){
//    StringBuilder sb = new StringBuilder("<h3>Unable to Parse</h3>");
//    if(!e.getMessage().isEmpty())sb.append("<div>" + e.getMessage() + "</div>");
//    if(e._failed != null && e._failed.length > 0){
//      sb.append("<div>\n<b>Found " + e._failed.length + " files which are not compatible with the given setup:</b></div>");
//      int n = e._failed.length;
//      if(n > 5){
//        sb.append("<div>" + e._failed[0] + "</div>");
//        sb.append("<div>" + e._failed[1] + "</div>");
//        sb.append("<div>...</div>");
//        sb.append("<div>" + e._failed[n-2] + "</div>");
//        sb.append("<div>" + e._failed[n-1] + "</div>");
//      } else for(int i = 0; i < n;++i)
//        sb.append("<div>" + e._failed[n-1] + "</div>");
//    } else if(e._gSetup == null || !e._gSetup.valid()) {
//      sb.append("Failed to find consistent parser setup for the given files!");
//    }
//    return sb.toString();
//  }

  protected static class PSetup {
    final transient ArrayList<Key> _keys;
    final transient Key [] _failedKeys;
    final CustomParser.PSetupGuess _setup;
    PSetup( ArrayList<Key> keys, Key [] fkeys, CustomParser.PSetupGuess pguess) { _keys=keys; _failedKeys = fkeys; _setup = pguess; }
  };
  // An H2O Key Query, which runs the basic CSV parsing heuristics.  Accepts
  // Key wildcards, and gathers all matching Keys for simultaneous parsing.
  // Multi-key parses are only allowed on compatible CSV files, and only 1 is
  // allowed to have headers.
  public class ExistingCSVKey extends TypeaheadInputText<PSetup> {
    public ExistingCSVKey(String name) {
      super(TypeaheadKeysRequest.class, name, true);
//      addPrerequisite(_parserType);
//      addPrerequisite(_separator);
    }

    @Override protected PSetup parse(String input) throws IllegalArgumentException {
      final Pattern p = makePattern(input);
      final Pattern exclude;
      if(_hdrFrom.specified())
        _header.setValue(true);
      exclude = _excludeExpression.specified()?makePattern(_excludeExpression.value()):null;

      // boolean badkeys = false;
      final Key [] keyAry = H2O.KeySnapshot.globalSnapshot().filter(new H2O.KVFilter() {
        @Override
        public boolean filter(H2O.KeyInfo k) {
          if(k._rawData) {
            String ks = k._key.toString();
            return (p.matcher(ks).matches() && (exclude == null || !exclude.matcher(ks).matches()));
          }
          return false;
        }<commit_msg>Updated parse ui to filter out empty files (so that user can see them not being included in the parse).<commit_after>package water.api;

import com.google.gson.JsonObject;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Set;
import java.util.regex.Pattern;
import water.*;
import water.parser.CsvParser;
import water.parser.CustomParser;
import water.parser.GuessSetup;
import water.util.RString;

abstract public class Parse extends Request {

  private   final ParserType     _parserType= new ParserType(PARSER_TYPE);
  private   final Separator      _separator = new Separator(SEPARATOR);
  private   final Bool           _header    = new Bool(HEADER,false,"Use first line as a header");
  protected final Bool           _sQuotes   = new Bool("single_quotes",false,"Enable single quotes as field quotation character");
  protected final HeaderKey      _hdrFrom   = new HeaderKey("header_from_file",false);
  protected final Str            _excludeExpression    = new Str("exclude","");
  protected final ExistingCSVKey _source    = new ExistingCSVKey(SOURCE_KEY);
  protected final NewH2OHexKey   _dest      = new NewH2OHexKey(DEST_KEY);
  protected final Bool           _blocking  = new Bool("blocking",false,"Synchronously wait until parse completes");
  @SuppressWarnings("unused")
  private   final Preview        _preview   = new Preview(PREVIEW);

  public Parse() {
    _excludeExpression.setRefreshOnChange();
    _header.setRefreshOnChange();
    _blocking._hideInQuery = true;
  }

//  private static String toHTML(ParseSetupGuessException e){
//    StringBuilder sb = new StringBuilder("<h3>Unable to Parse</h3>");
//    if(!e.getMessage().isEmpty())sb.append("<div>" + e.getMessage() + "</div>");
//    if(e._failed != null && e._failed.length > 0){
//      sb.append("<div>\n<b>Found " + e._failed.length + " files which are not compatible with the given setup:</b></div>");
//      int n = e._failed.length;
//      if(n > 5){
//        sb.append("<div>" + e._failed[0] + "</div>");
//        sb.append("<div>" + e._failed[1] + "</div>");
//        sb.append("<div>...</div>");
//        sb.append("<div>" + e._failed[n-2] + "</div>");
//        sb.append("<div>" + e._failed[n-1] + "</div>");
//      } else for(int i = 0; i < n;++i)
//        sb.append("<div>" + e._failed[n-1] + "</div>");
//    } else if(e._gSetup == null || !e._gSetup.valid()) {
//      sb.append("Failed to find consistent parser setup for the given files!");
//    }
//    return sb.toString();
//  }

  protected static class PSetup {
    final transient ArrayList<Key> _keys;
    final transient Key [] _failedKeys;
    final CustomParser.PSetupGuess _setup;
    PSetup( ArrayList<Key> keys, Key [] fkeys, CustomParser.PSetupGuess pguess) { _keys=keys; _failedKeys = fkeys; _setup = pguess; }
  };
  // An H2O Key Query, which runs the basic CSV parsing heuristics.  Accepts
  // Key wildcards, and gathers all matching Keys for simultaneous parsing.
  // Multi-key parses are only allowed on compatible CSV files, and only 1 is
  // allowed to have headers.
  public class ExistingCSVKey extends TypeaheadInputText<PSetup> {
    public ExistingCSVKey(String name) {
      super(TypeaheadKeysRequest.class, name, true);
//      addPrerequisite(_parserType);
//      addPrerequisite(_separator);
    }

    @Override protected PSetup parse(String input) throws IllegalArgumentException {
      final Pattern p = makePattern(input);
      final Pattern exclude;
      if(_hdrFrom.specified())
        _header.setValue(true);
      exclude = _excludeExpression.specified()?makePattern(_excludeExpression.value()):null;

      // boolean badkeys = false;
      final Key [] keyAry = H2O.KeySnapshot.globalSnapshot().filter(new H2O.KVFilter() {
        @Override
        public boolean filter(H2O.KeyInfo k) {
          if(k._rawData && k._nrows > 0) {
            String ks = k._key.toString();
            return (p.matcher(ks).matches() && (exclude == null || !exclude.matcher(ks).matches()));
          }
          return false;
        }
{'fraction_non_alphanumeric': 0.1362394436164928, 'fraction_numerical': 0.004346746150024838, 'mean_word_length': 3.1639089968976215, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '801612', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2397, 'n_words': 631}