<commit_before>package water.api;

import java.util.ArrayList;
import java.util.Collections;
import java.util.regex.Pattern;

import water.*;
import water.fvec.Frame;
import water.parser.*;
import water.util.RString;
import water.util.Utils;

import com.google.gson.JsonObject;

public class Parse extends Request {

  private   final ParserType _parserType    = new ParserType(PARSER_TYPE);
  private   final Separator      _separator = new Separator(SEPARATOR);
  private   final Bool           _header    = new Bool(HEADER,false,"Use first line as a header");
  protected final H2OExistingKey  _hdrFrom  = new H2OExistingKey("header_from_file",false);
  protected final Str _excludeExpression    = new Str("exclude","");
  protected final ExistingCSVKey _source    = new ExistingCSVKey(SOURCE_KEY);
  protected final NewH2OHexKey   _dest      = new NewH2OHexKey(DEST_KEY);
  @SuppressWarnings("unused")
  private   final Preview        _preview   = new Preview(PREVIEW);

  public Parse() {
    _hdrFrom.addPrerequisite(_header);
    _excludeExpression.setRefreshOnChange();}


  protected CustomParser.ParserSetup guessSetup(ArrayList<Key> keys,Key headerKey, CustomParser.ParserSetup setup, boolean checkHeader){
    String [] colNames = null;
    CustomParser.ParserSetup gSetup = null, hSetup = null;
    if(headerKey != null){ // handle the header file!
      Value v = DKV.get(headerKey);
      if(!v.isRawData()){ // either ValueArray or a Frame, just extract the headers
        if(v.isArray()){
          ValueArray ary = v.get();
          colNames = ary.colNames();
        } else if(v.isFrame()){
          Frame fr = v.get();
          colNames = fr._names;
        } else
          throw new IllegalArgumentException("Headers can only come from unparsed data, ValueArray or a frame. Got " + v.newInstance().getClass().getSimpleName());
      } else { // check the hdr setup by parsing first bytes
        hSetup = ParseDataset.guessSetup(Utils.getFirstUnzipedBytes(headerKey),setup,checkHeader);
        if(hSetup._data != null && hSetup._data.length > 1){ // the hdr file had both hdr and data, it better be part of the parse and represent the global parser setup
          if(!keys.contains(headerKey))
            throw new IllegalArgumentException("Header file must either be part of the parse or contain only header (no data!).");
          gSetup = hSetup; // else we got a global setup file all other files must comply with!
        }
        colNames = hSetup._columnNames;
      }
    }
    if(gSetup == null){ // we did not get global setup
      Key k = keys.get(0);
      if(headerKey != null && k.equals(headerKey) && keys.size() > 1)
        k = keys.get(1);
      gSetup = ParseDataset.guessSetup(Utils.getFirstUnzipedBytes(k), new CustomParser.ParserSetup(), checkHeader && headerKey == null);
      if(colNames.length != gSetup._ncols) throw new IllegalArgumentException("number of columns in the header file and the parsed files don't match!");
    }
    // now set the header info in the final setup
    if(colNames != null){
      gSetup._header = true;
      gSetup._columnNames = colNames;
    } else
      gSetup._header = false;
    return gSetup;

  }


  protected static class PSetup {
    final transient ArrayList<Key> _keys;
    final CustomParser.ParserSetup _setup;
    PSetup( ArrayList<Key> keys, CustomParser.ParserSetup parser) { _keys=keys; _setup = parser; }
  };
  // An H2O Key Query, which runs the basic CSV parsing heuristics.  Accepts
  // Key wildcards, and gathers all matching Keys for simultaneous parsing.
  // Multi-key parses are only allowed on compatible CSV files, and only 1 is
  // allowed to have headers.
  public class ExistingCSVKey extends TypeaheadInputText<PSetup> {
    public ExistingCSVKey(String name) {
      super(TypeaheadKeysRequest.class, name, true);
//      addPrerequisite(_parserType);
//      addPrerequisite(_separator);
    }


    @Override protected PSetup parse(String input) throws IllegalArgumentException {
      Pattern p = makePattern(input);
      Pattern exclude = null;
      if(_excludeExpression.specified())
        exclude = makePattern(_excludeExpression.value());
      ArrayList<Key> keys = new ArrayList();
     // boolean badkeys = false;
      for( Key key : H2O.keySet() ) { // For all keys
        if( !key.user_allowed() ) continue;
        String ks = key.toString();
        if( !p.matcher(ks).matches() ) // Ignore non-matching keys
          continue;
        if(exclude != null && exclude.matcher(ks).matches())
          continue;
        Value v2 = DKV.get(key);  // Look at it
        if(v2.isHex())// filter common mistake such as *filename* with filename.hex already present
          continue;
        keys.add(key);        // Add to list
      }
      if(keys.size() == 0 )
        throw new IllegalArgumentException("I did not find any keys matching this pattern!");
      Collections.sort(keys);   // Sort all the keys, except the 1 header guy
      // now we assume the first key has the header
      Key hKey = keys.get(0);
      if(_hdrFrom.specified())
        hKey = _hdrFrom.value()._key;
      else
        _hdrFrom.setValue(DKV.get(hKey));
      CustomParser.ParserSetup setup = guessSetup(keys, hKey, new CustomParser.ParserSetup(_parserType.value(),_separator.value(),_header.specified()?_header.value():false),_header.specified());
      if(setup == null)
        throw new IllegalArgumentException("I cannot figure out this file; Please select the parse setup manually.");
      if(_header.specified())
        setup.setHeader(_header.value());
      PSetup res = new PSetup(keys,setup);
      _parserType.setValue(res._setup._pType);
      _separator.setValue(res._setup._separator);
      if(!_header.specified())
        _header.setValue(res._setup._header);
      _header._hideInQuery = _separator._hideInQuery = res._setup._pType != CustomParser.ParserType.CSV;
      return res;
    }

    private final String keyRow(Key k){
      return "<tr><td>" + k + "</td></tr>\n";
    }

    @Override
    public String queryComment(){
      if(!specified())return "";
      PSetup p = value();
      StringBuilder sb = new StringBuilder();
      if(p._keys.size() <= 10){
        for(Key k:p._keys)
          sb.append(keyRow(k));
      } else {
        int n = p._keys.size();
        for(int i = 0; i < 5; ++i)
          sb.append(keyRow(p._keys.get(i)));
        sb.append("<tr><td>...</td></tr>\n");
        for(int i = 5; i > 0; --i)
          sb.append(keyRow(p._keys.get(n-i)));
      }
      return
          "<div class='alert'><b> Found " + p._keys.size() +  " files matching the expression.</b><br/>\n" +
          "<table>\n" +
           sb.toString() +
          "</table></div>";
    }

    private Pattern makePattern(String input) {
      // Reg-Ex pattern match all keys, like file-globbing.
      // File-globbing: '?' allows an optional single character, regex needs '.?'
      // File-globbing: '*' allows any characters, regex needs '*?'
      // File-globbing: '\' is normal character in windows, regex needs '\\'
      String patternStr = input.replace("?",".?").replace("*",".*?").replace("\\","\\\\").replace("(","\\(").replace(")","\\)");
      Pattern p = Pattern.compile(patternStr);
      return p;
    }

    @Override protected PSetup defaultValue() { return null; }
    @Override protected String queryDescription() { return "An existing H2O key (or regex of keys) of CSV text"; }
  }


  // A Query String, which defaults to the source Key with a '.hex' suffix
  protected class NewH2OHexKey extends Str {
    NewH2OHexKey(String name) {
      super(name,null/*not required flag*/);
      addPrerequisite(_source);
    }
    @Override protected String defaultValue() {
      PSetup setup = _source.value();
      if( setup == null ) return null;
      String n = setup._keys.get(0).toString();
      int dot = n.lastIndexOf('.');
      if( dot > 0 ) n = n.substring(0, dot);
      int i = 0;
      String res = n + Extensions.HEX;
      Key k = Key.make(res);
      while(DKV.get(k) != null)
        k = Key.make(res = n + ++i + Extensions.HEX);
      return res;
    }
    @Override protected String queryDescription() { return "Destination hex key"; }
  }

  // A Query Bool, which includes a pretty HTML-ized version of the first few
  // parsed data rows.  If the value() is TRUE, we display as-if the first row
  // is a label/header column, and if FALSE not.
  public class Preview extends Argument {
      Preview(String name) {
      super(name,false);
      addPrerequisite(_source);
      addPrerequisite(_separator);
      addPrerequisite(_parserType);
      addPrerequisite(_header);
      setRefreshOnChange();
    }
    @Override protected String queryElement() {
      // first determine the value to put in the field
      // if no original value was supplied, use the provided one
      String[][] data = null;
      PSetup psetup = _source.value();
      StringBuilder sb = new StringBuilder("<div class='alert'>" + psetup._setup.toString() + "</div>");
      if(psetup._setup != null)
        data = psetup._setup._data;
      String [] header = psetup._setup._columnNames;
      if( data != null ) {
        sb.append("<table class='table table-striped table-bordered'>");
        int j = 0;
        if( psetup._setup._header && header != null) { // Obvious header display, if asked for
          sb.append("<tr><th>Row#</th>");
          for( String s : header ) sb.append("<th>").append(s).append("</th>");
          sb.append("</tr>");
          if(header == data[0]) ++j;
        }<commit_msg>pick header file update. Display header right under the box with header file.<commit_after>package water.api;

import java.util.ArrayList;
import java.util.Collections;
import java.util.regex.Pattern;

import water.*;
import water.fvec.Frame;
import water.parser.*;
import water.util.RString;
import water.util.Utils;

import com.google.gson.JsonObject;

public class Parse extends Request {

  private   final ParserType _parserType    = new ParserType(PARSER_TYPE);
  private   final Separator      _separator = new Separator(SEPARATOR);
  private   final Bool           _header    = new Bool(HEADER,false,"Use first line as a header");
  protected final HeaderKey      _hdrFrom   = new HeaderKey("header_from_file",false);
  protected final Str _excludeExpression    = new Str("exclude","");
  protected final ExistingCSVKey _source    = new ExistingCSVKey(SOURCE_KEY);
  protected final NewH2OHexKey   _dest      = new NewH2OHexKey(DEST_KEY);
  @SuppressWarnings("unused")
  private   final Preview        _preview   = new Preview(PREVIEW);

  public Parse() {
    _hdrFrom.addPrerequisite(_header);
    _excludeExpression.setRefreshOnChange();}


  protected CustomParser.ParserSetup guessSetup(ArrayList<Key> keys,Key headerKey, CustomParser.ParserSetup setup, boolean checkHeader){
    String [] colNames = null;
    CustomParser.ParserSetup gSetup = null, hSetup = null;
    if(headerKey != null){ // handle the header file!
      Value v = DKV.get(headerKey);
      if(!v.isRawData()){ // either ValueArray or a Frame, just extract the headers
        if(v.isArray()){
          ValueArray ary = v.get();
          colNames = ary.colNames();
        } else if(v.isFrame()){
          Frame fr = v.get();
          colNames = fr._names;
        } else
          throw new IllegalArgumentException("Headers can only come from unparsed data, ValueArray or a frame. Got " + v.newInstance().getClass().getSimpleName());
      } else { // check the hdr setup by parsing first bytes
        hSetup = ParseDataset.guessSetup(Utils.getFirstUnzipedBytes(headerKey),setup,checkHeader);
        if(hSetup._data != null && hSetup._data.length > 1){ // the hdr file had both hdr and data, it better be part of the parse and represent the global parser setup
          if(!keys.contains(headerKey))
            throw new IllegalArgumentException("Header file must either be part of the parse or contain only header (no data!).");
          gSetup = hSetup; // else we got a global setup file all other files must comply with!
        }
        colNames = hSetup._columnNames;
      }
    }
    if(gSetup == null){ // we did not get global setup
      Key k = keys.get(0);
      if(headerKey != null && k.equals(headerKey) && keys.size() > 1)
        k = keys.get(1);
      gSetup = ParseDataset.guessSetup(Utils.getFirstUnzipedBytes(k), new CustomParser.ParserSetup(), checkHeader && headerKey == null);
      if(colNames.length != gSetup._ncols) throw new IllegalArgumentException("number of columns in the header file and the parsed files don't match!");
    }
    // now set the header info in the final setup
    if(colNames != null){
      gSetup._header = true;
      gSetup._columnNames = colNames;
    } else
      gSetup._header = false;
    return gSetup;

  }


  protected static class PSetup {
    final transient ArrayList<Key> _keys;
    final CustomParser.ParserSetup _setup;
    PSetup( ArrayList<Key> keys, CustomParser.ParserSetup parser) { _keys=keys; _setup = parser; }
  };
  // An H2O Key Query, which runs the basic CSV parsing heuristics.  Accepts
  // Key wildcards, and gathers all matching Keys for simultaneous parsing.
  // Multi-key parses are only allowed on compatible CSV files, and only 1 is
  // allowed to have headers.
  public class ExistingCSVKey extends TypeaheadInputText<PSetup> {
    public ExistingCSVKey(String name) {
      super(TypeaheadKeysRequest.class, name, true);
//      addPrerequisite(_parserType);
//      addPrerequisite(_separator);
    }


    @Override protected PSetup parse(String input) throws IllegalArgumentException {
      Pattern p = makePattern(input);
      Pattern exclude = null;
      if(_excludeExpression.specified())
        exclude = makePattern(_excludeExpression.value());
      ArrayList<Key> keys = new ArrayList();
     // boolean badkeys = false;
      for( Key key : H2O.keySet() ) { // For all keys
        if( !key.user_allowed() ) continue;
        String ks = key.toString();
        if( !p.matcher(ks).matches() ) // Ignore non-matching keys
          continue;
        if(exclude != null && exclude.matcher(ks).matches())
          continue;
        Value v2 = DKV.get(key);  // Look at it
        if(v2.isHex())// filter common mistake such as *filename* with filename.hex already present
          continue;
        keys.add(key);        // Add to list
      }
      if(keys.size() == 0 )
        throw new IllegalArgumentException("I did not find any keys matching this pattern!");
      Collections.sort(keys);   // Sort all the keys, except the 1 header guy
      // now we assume the first key has the header
      Key hKey = keys.get(0);
      if(_hdrFrom.specified())
        hKey = _hdrFrom.value()._key;
      else
        _hdrFrom.setValue(DKV.get(hKey));
      CustomParser.ParserSetup setup = guessSetup(keys, hKey, new CustomParser.ParserSetup(_parserType.value(),_separator.value(),_header.specified()?_header.value():false),_header.specified());
      if(setup == null)
        throw new IllegalArgumentException("I cannot figure out this file; Please select the parse setup manually.");
      if(!_header.value()){
        setup.setHeader(_header.value());
        if(!_header.value()) _hdrFrom.disable("Header is disabled.");
      }
      PSetup res = new PSetup(keys,setup);
      _parserType.setValue(res._setup._pType);
      _separator.setValue(res._setup._separator);
      if(!_header.specified())
        _header.setValue(res._setup._header);
      _header._hideInQuery = _separator._hideInQuery = res._setup._pType != CustomParser.ParserType.CSV;
      return res;
    }

    private final String keyRow(Key k){
      return "<tr><td>" + k + "</td></tr>\n";
    }

    @Override
    public String queryComment(){
      if(!specified())return "";
      PSetup p = value();
      StringBuilder sb = new StringBuilder();
      if(p._keys.size() <= 10){
        for(Key k:p._keys)
          sb.append(keyRow(k));
      } else {
        int n = p._keys.size();
        for(int i = 0; i < 5; ++i)
          sb.append(keyRow(p._keys.get(i)));
        sb.append("<tr><td>...</td></tr>\n");
        for(int i = 5; i > 0; --i)
          sb.append(keyRow(p._keys.get(n-i)));
      }
      return
          "<div class='alert'><b> Found " + p._keys.size() +  " files matching the expression.</b><br/>\n" +
          "<table>\n" +
           sb.toString() +
          "</table></div>";
    }

    private Pattern makePattern(String input) {
      // Reg-Ex pattern match all keys, like file-globbing.
      // File-globbing: '?' allows an optional single character, regex needs '.?'
      // File-globbing: '*' allows any characters, regex needs '*?'
      // File-globbing: '\' is normal character in windows, regex needs '\\'
      String patternStr = input.replace("?",".?").replace("*",".*?").replace("\\","\\\\").replace("(","\\(").replace(")","\\)");
      Pattern p = Pattern.compile(patternStr);
      return p;
    }

    @Override protected PSetup defaultValue() { return null; }
    @Override protected String queryDescription() { return "An existing H2O key (or regex of keys) of CSV text"; }
  }


  // A Query String, which defaults to the source Key with a '.hex' suffix
  protected class NewH2OHexKey extends Str {
    NewH2OHexKey(String name) {
      super(name,null/*not required flag*/);
      addPrerequisite(_source);
    }
    @Override protected String defaultValue() {
      PSetup setup = _source.value();
      if( setup == null ) return null;
      String n = setup._keys.get(0).toString();
      int dot = n.lastIndexOf('.');
      if( dot > 0 ) n = n.substring(0, dot);
      int i = 0;
      String res = n + Extensions.HEX;
      Key k = Key.make(res);
      while(DKV.get(k) != null)
        k = Key.make(res = n + ++i + Extensions.HEX);
      return res;
    }
    @Override protected String queryDescription() { return "Destination hex key"; }
  }
  public class HeaderKey extends H2OExistingKey {
    public HeaderKey(String name, boolean required) {
      super(name, required);
    }
    @Override protected String queryElement() {
      String [] colnames = _source.value()._setup._columnNames;
      StringBuilder sb = new StringBuilder(super.queryElement() + "\n");
      if(colnames != null){
        sb.append("<table class='table table-striped table-bordered'>").append("<tr><th>Header:</th>");
        for( String s : colnames ) sb.append("<th>").append(s).append("</th>");
        sb.append("</tr></table>");
      }
      return sb.toString();
    }

  }
  // A Query Bool, which includes a pretty HTML-ized version of the first few
  // parsed data rows.  If the value() is TRUE, we display as-if the first row
  // is a label/header column, and if FALSE not.
  public class Preview extends Argument {
      Preview(String name) {
      super(name,false);
      addPrerequisite(_source);
      addPrerequisite(_separator);
      addPrerequisite(_parserType);
      addPrerequisite(_header);
      setRefreshOnChange();
    }
    @Override protected String queryElement() {
      // first determine the value to put in the field
      // if no original value was supplied, use the provided one
      String[][] data = null;
      PSetup psetup = _source.value();
      StringBuilder sb = new StringBuilder("<div class='alert'>" + psetup._setup.toString() + "</div>");
      if(psetup._setup != null)
        data = psetup._setup._data;
      String [] header = psetup._setup._columnNames;
      if( data != null ) {
        sb.append("<table class='table table-striped table-bordered'>");
        int j = 0;
        if( psetup._setup._header && header != null) { // Obvious header display, if asked for
          sb.append("<tr><th>Row#</th>");
          for( String s : header ) sb.append("<th>").append(s).append("</th>");
          sb.append("</tr>");
          if(header == data[0]) ++j;
        }
{'fraction_non_alphanumeric': 0.1282655568369854, 'fraction_numerical': 0.00305871734443163, 'mean_word_length': 2.793077215671358, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3812030', 'n_tokens_mistral': 5839, 'n_tokens_neox': 5735, 'n_words': 1684}