<commit_before>package water.parser;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;

import org.apache.poi.ss.formula.ptg.PowerPtg;

import water.Iced;

/**
 * Parser for SVM light format.
 * @author tomasnykodym
 *
 */
public class SVMLightParser extends CustomParser{
  final boolean isWhiteSpace(char c){return c == ' '  || c == '\t';}
  final boolean isNewLine(char c)   {return c == '\r' || c == '\n';}
  private static final byte SKIP_LINE = 0;
  private static final byte EXPECT_COND_LF = 1;
  private static final byte EOL = 2;
  private static final byte TOKEN = 3;
  private static final byte SKIP_TOKEN = 4;
  private static final byte NUMBER = 5;
  private static final byte NUMBER_FRACTION = 6;
  private static final byte NUMBER_EXP = 7;
  private static final byte INVALID_NUMBER = 8;
  private static final byte NUMBER_EXP_START = 9;
  private static final byte NUMBER_END = 10;
  private static final byte WHITESPACE_BEFORE_TOKEN = 11;
  private static final byte POSSIBLE_EMPTY_LINE = 12;
  private static final byte QID0 = 13;
  private static final byte QID1 = 14;

  // line global states
  private static final int TGT = 1;
  private static final int COL = 2;
  private static final int VAL = 3;

  private static final long LARGEST_DIGIT_NUMBER = 1000000000000000000L;

  final static char COMMENT_CHR = '#';
  final static char DECIMAL_SEP = '.';
  int _failedLines = 0;

  public SVMLightParser() {super(new ParserSetup(ParserType.SVMLight, CsvParser.AUTO_SEP, false));}
  public SVMLightParser(ParserSetup setup) {super(setup);}
  @Override
  public SVMLightParser clone(){return new SVMLightParser(_setup);}
  @Override
  public boolean parallelParseSupported(){return true;}

  /**
   * Try to parse the bits as svm light format, return SVMParser instance if the input is in svm light format, null otherwise.
   * @param bits
   * @return SVMLightPArser instance or null
   */
  public static PSetupGuess guessSetup(byte [] bits){
    // find the last eof
    int i = bits.length-1;
    while(bits[i] != '\n' && i >= 0)--i;
    assert i > 0;
    InputStream is = new ByteArrayInputStream(Arrays.copyOf(bits,i));
    SVMLightParser p = new SVMLightParser(new ParserSetup(ParserType.SVMLight, CsvParser.AUTO_SEP, false));
    InspectDataOut dout = new InspectDataOut();
    try{p.streamParse(is, dout);}catch(Exception e){throw new RuntimeException(e);}
    return new PSetupGuess(new ParserSetup(ParserType.SVMLight, CsvParser.AUTO_SEP, dout._ncols,false,null,false),dout._nlines,dout._invalidLines,dout.data(),dout._ncols > 0 && dout._nlines > 0 && dout._nlines > dout._invalidLines,dout.errors());
  }
  @Override<commit_msg>Fix NPE in SVMLightParser.<commit_after>package water.parser;

import water.Iced;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;

/**
 * Parser for SVM light format.
 * @author tomasnykodym
 *
 */
public class SVMLightParser extends CustomParser{
  final boolean isWhiteSpace(char c){return c == ' '  || c == '\t';}
  final boolean isNewLine(char c)   {return c == '\r' || c == '\n';}
  private static final byte SKIP_LINE = 0;
  private static final byte EXPECT_COND_LF = 1;
  private static final byte EOL = 2;
  private static final byte TOKEN = 3;
  private static final byte SKIP_TOKEN = 4;
  private static final byte NUMBER = 5;
  private static final byte NUMBER_FRACTION = 6;
  private static final byte NUMBER_EXP = 7;
  private static final byte INVALID_NUMBER = 8;
  private static final byte NUMBER_EXP_START = 9;
  private static final byte NUMBER_END = 10;
  private static final byte WHITESPACE_BEFORE_TOKEN = 11;
  private static final byte POSSIBLE_EMPTY_LINE = 12;
  private static final byte QID0 = 13;
  private static final byte QID1 = 14;

  // line global states
  private static final int TGT = 1;
  private static final int COL = 2;
  private static final int VAL = 3;

  private static final long LARGEST_DIGIT_NUMBER = 1000000000000000000L;

  final static char COMMENT_CHR = '#';
  final static char DECIMAL_SEP = '.';
  int _failedLines = 0;

  public SVMLightParser() {super(new ParserSetup(ParserType.SVMLight, CsvParser.AUTO_SEP, false));}
  public SVMLightParser(ParserSetup setup) {super(setup);}
  @Override
  public SVMLightParser clone(){return new SVMLightParser(_setup);}
  @Override
  public boolean parallelParseSupported(){return true;}

  /**
   * Try to parse the bytes as svm light format, return SVMParser instance if the input is in svm light format, null otherwise.
   * @param bytes
   * @return SVMLightPArser instance or null
   */
  public static PSetupGuess guessSetup(byte [] bytes){
    // find the last eof
    int i = bytes.length-1;
    while(i > 0 && bytes[i] != '\n')--i;
    assert i >= 0;
    InputStream is = new ByteArrayInputStream(Arrays.copyOf(bytes,i));
    SVMLightParser p = new SVMLightParser(new ParserSetup(ParserType.SVMLight, CsvParser.AUTO_SEP, false));
    InspectDataOut dout = new InspectDataOut();
    try{p.streamParse(is, dout);}catch(Exception e){throw new RuntimeException(e);}
    return new PSetupGuess(new ParserSetup(ParserType.SVMLight, CsvParser.AUTO_SEP, dout._ncols,false,null,false),dout._nlines,dout._invalidLines,dout.data(),dout._ncols > 0 && dout._nlines > 0 && dout._nlines > dout._invalidLines,dout.errors());
  }
  @Override
{'fraction_non_alphanumeric': 0.09957588050894339, 'fraction_numerical': 0.018439977872026555, 'mean_word_length': 4.286549707602339, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2004246', 'n_tokens_mistral': 1844, 'n_tokens_neox': 1734, 'n_words': 542}