<commit_before>package water.parser;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;


import water.*;
import water.fvec.ParseDataset2.ParseProgressMonitor;
import water.parser.ParseDataset.ParseSetupGuessException;


public abstract class CustomParser extends Iced {
  public static final byte CHAR_TAB = '\t';
  public static final byte CHAR_LF = 10;
  public static final byte CHAR_SPACE = ' ';
  public static final byte CHAR_CR = 13;
  public static final byte CHAR_VT = 11;
  public static final byte CHAR_FF = 12;
  public static final byte CHAR_DOUBLE_QUOTE = '"';
  public static final byte CHAR_SINGLE_QUOTE = '\'';
  public static final byte CHAR_NULL = 0;
  public static final byte CHAR_COMMA = ',';

  public final static int MAX_PREVIEW_COLS  = 100;
  public final static int MAX_PREVIEW_LINES = 50;

  public final ParserSetup _setup;

  public CustomParser(ParserSetup setup){_setup = setup;}

  public static class PSetupGuess extends Iced {
    public final ParserSetup _setup;
    public final int _invalidLines;
    public final int _validLines;
    public final String []   _errors;
    public Key _setupFromFile;
    public Key _hdrFromFile;
    public String [][] _data;
    public PSetupGuess(ParserSetup ps, int vlines, int ilines, String [][] data, String [] errors){
      _setup = ps;
      _invalidLines = ilines;
      _validLines = vlines;
      _errors = errors;
      _data = data;
    }

    public void checkColumnNames(){
      _setup.checkColumnNames();
    }

    public final boolean valid(){
      return _setup._ncols > 0 && _validLines > 0 && _invalidLines < _validLines;
    }
    public final boolean hasErrors(){
      return _errors != null && _errors.length > 0;
    }
    public String toString(){
      if(!valid())
        return "Parser setup appears to be broken, got " + _setup.toString();
      else if(hasErrors())
        return "Parser setup appears to work with some errors, got " + _setup.toString();
      else
        return "Parser setup working fine, got " + _setup.toString();
    }
  }
  public enum ParserType {
    AUTO(false),XLS(false),XLSX(false),CSV(true), SVMLight(true);
    public final boolean parallelParseSupported;
    ParserType(boolean par){parallelParseSupported = par;}
  }
  public static class ParserSetup extends Iced implements Cloneable{
    public final ParserType _pType;
    public final byte _separator;
    public boolean _header;
    public boolean _singleQuotes;
    public String [] _columnNames;
    public final int _ncols;

    public ParserSetup() {
      _pType = ParserType.AUTO;
      _separator = CsvParser.AUTO_SEP;
      _header = false;
      _ncols = 0;
      _columnNames = null;
    }
    protected ParserSetup(ParserType t) {
      this(t,CsvParser.AUTO_SEP,0,false,null,false);
    }
    public ParserSetup(ParserType t, byte sep, boolean header) {
      _pType = t;
      _separator = sep;
      _header = header;
      _columnNames = null;
      _ncols = 0;
    }
    public ParserSetup(ParserType t, byte sep, int ncolumns, boolean header, String [] columnNames, boolean singleQuotes) {
      _pType = t;
      _separator = sep;
      _ncols = ncolumns;
      _header = header;
      _columnNames = columnNames;
      _singleQuotes = singleQuotes;
    }
    public void checkColumnNames(){
      if(_header){
        HashSet<String> uniqueNames = new HashSet<String>();
        HashSet<String> conflictingNames = new HashSet<String>();
        for(String n:_columnNames){
          if(!uniqueNames.contains(n)){
            uniqueNames.add(n);
          } else {
            conflictingNames.add(n);
          }
        }
        if(!conflictingNames.isEmpty())
          throw new ParseSetupGuessException("Invalid header. Got conflicting column names. " + conflictingNames.toString(),null,null);
      }
    }
    public ParserSetup clone(){
      return new ParserSetup(_pType, _separator, _ncols,_header,null,_singleQuotes);
    }
    public boolean isCompatible(ParserSetup other){
      if(other == null || _pType != other._pType)return false;
      if(_pType == ParserType.CSV && (_separator != other._separator || _ncols != other._ncols))
        return false;
      return true;
    }
    public CustomParser parser(){
      switch(this._pType){
        case CSV:
          return new CsvParser(this);
        case SVMLight:
          return new SVMLightParser(this);
        case XLS:
          return new XlsParser(this);
        default:
          throw H2O.unimpl();
      }
    }
    public String toString(){
      StringBuilder sb = new StringBuilder(_pType.name());
      switch(_pType){
        case SVMLight:
          sb.append(" data with (estimated) " + _ncols + " columns.");<commit_msg>Better error msg for repeated column labels<commit_after>package water.parser;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;


import water.*;
import water.fvec.ParseDataset2.ParseProgressMonitor;
import water.parser.ParseDataset.ParseSetupGuessException;


public abstract class CustomParser extends Iced {
  public static final byte CHAR_TAB = '\t';
  public static final byte CHAR_LF = 10;
  public static final byte CHAR_SPACE = ' ';
  public static final byte CHAR_CR = 13;
  public static final byte CHAR_VT = 11;
  public static final byte CHAR_FF = 12;
  public static final byte CHAR_DOUBLE_QUOTE = '"';
  public static final byte CHAR_SINGLE_QUOTE = '\'';
  public static final byte CHAR_NULL = 0;
  public static final byte CHAR_COMMA = ',';

  public final static int MAX_PREVIEW_COLS  = 100;
  public final static int MAX_PREVIEW_LINES = 50;

  public final ParserSetup _setup;

  public CustomParser(ParserSetup setup){_setup = setup;}

  public static class PSetupGuess extends Iced {
    public final ParserSetup _setup;
    public final int _invalidLines;
    public final int _validLines;
    public final String []   _errors;
    public Key _setupFromFile;
    public Key _hdrFromFile;
    public String [][] _data;
    public PSetupGuess(ParserSetup ps, int vlines, int ilines, String [][] data, String [] errors){
      _setup = ps;
      _invalidLines = ilines;
      _validLines = vlines;
      _errors = errors;
      _data = data;
    }

    public void checkColumnNames(){
      _setup.checkColumnNames();
    }

    public final boolean valid(){
      return _setup._ncols > 0 && _validLines > 0 && _invalidLines < _validLines;
    }
    public final boolean hasErrors(){
      return _errors != null && _errors.length > 0;
    }
    public String toString(){
      if(!valid())
        return "Parser setup appears to be broken, got " + _setup.toString();
      else if(hasErrors())
        return "Parser setup appears to work with some errors, got " + _setup.toString();
      else
        return "Parser setup working fine, got " + _setup.toString();
    }
  }
  public enum ParserType {
    AUTO(false),XLS(false),XLSX(false),CSV(true), SVMLight(true);
    public final boolean parallelParseSupported;
    ParserType(boolean par){parallelParseSupported = par;}
  }
  public static class ParserSetup extends Iced implements Cloneable{
    public final ParserType _pType;
    public final byte _separator;
    public boolean _header;
    public boolean _singleQuotes;
    public String [] _columnNames;
    public final int _ncols;

    public ParserSetup() {
      _pType = ParserType.AUTO;
      _separator = CsvParser.AUTO_SEP;
      _header = false;
      _ncols = 0;
      _columnNames = null;
    }
    protected ParserSetup(ParserType t) {
      this(t,CsvParser.AUTO_SEP,0,false,null,false);
    }
    public ParserSetup(ParserType t, byte sep, boolean header) {
      _pType = t;
      _separator = sep;
      _header = header;
      _columnNames = null;
      _ncols = 0;
    }
    public ParserSetup(ParserType t, byte sep, int ncolumns, boolean header, String [] columnNames, boolean singleQuotes) {
      _pType = t;
      _separator = sep;
      _ncols = ncolumns;
      _header = header;
      _columnNames = columnNames;
      _singleQuotes = singleQuotes;
    }
    public void checkColumnNames(){
      if(_header){
        HashSet<String> uniqueNames = new HashSet<String>();
        HashSet<String> conflictingNames = new HashSet<String>();
        for(String n:_columnNames){
          if(!uniqueNames.contains(n)){
            uniqueNames.add(n);
          } else {
            conflictingNames.add(n);
          }
        }
        if(!conflictingNames.isEmpty())
          throw new ParseSetupGuessException("Column labels must be unique but these labels are repeated:" + conflictingNames.toString(),null,null);
      }
    }
    public ParserSetup clone(){
      return new ParserSetup(_pType, _separator, _ncols,_header,null,_singleQuotes);
    }
    public boolean isCompatible(ParserSetup other){
      if(other == null || _pType != other._pType)return false;
      if(_pType == ParserType.CSV && (_separator != other._separator || _ncols != other._ncols))
        return false;
      return true;
    }
    public CustomParser parser(){
      switch(this._pType){
        case CSV:
          return new CsvParser(this);
        case SVMLight:
          return new SVMLightParser(this);
        case XLS:
          return new XlsParser(this);
        default:
          throw H2O.unimpl();
      }
    }
    public String toString(){
      StringBuilder sb = new StringBuilder(_pType.name());
      switch(_pType){
        case SVMLight:
          sb.append(" data with (estimated) " + _ncols + " columns.");
{'fraction_non_alphanumeric': 0.09328866087320699, 'fraction_numerical': 0.00460684745052874, 'mean_word_length': 2.895595432300163, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2225654', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2744, 'n_words': 805}