<commit_before>package com.orientechnologies.common.console;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.CharBuffer;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

/**
 * @author <a href='mailto:<EMAIL>'> <NAME> </a>
 */
public class ODFACommandStream implements OCommandStream {
  public static final int      BUFFER_SIZE = 1024;

  private Reader               reader;
  private CharBuffer           buffer;
  private final Set<Character> separators  = new HashSet<Character>(Arrays.asList(';', '\n'));
  private int                  position;
  private int                  start;
  private int                  end;
  private StringBuilder        partialResult;
  private State                state;

  public ODFACommandStream(String commands) {
    reader = new StringReader(commands);
    init();
  }

  public ODFACommandStream(File file) throws FileNotFoundException {
    reader = new BufferedReader(new FileReader(file));
    init();
  }

  private void init() {
    buffer = CharBuffer.allocate(BUFFER_SIZE);
    buffer.flip();
  }

  @Override
  public boolean hasNext() {
    try {
      fillBuffer();

      return buffer.hasRemaining();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private void fillBuffer() throws IOException {
    if (!buffer.hasRemaining()) {
      buffer.clear();
      reader.read(buffer);
      buffer.flip();
    }
  }

  @Override
  public String nextCommand() {
    try {
      fillBuffer();

      partialResult = new StringBuilder();
      state = State.S;
      start = 0;
      end = -1;
      position = 0;
      Symbol s = null;
      while (state != State.E) {
        s = nextSymbol();

        final State newState = transition(state, s);

        if (state == State.S && newState != State.S)
          start = position;

        if (newState == State.A)
          end = position;

        if (newState == State.F)
          throw new IllegalStateException("Unexpected end of file");

        state = newState;
        position++;
      }

      if (s == Symbol.EOF) {
        position--;
        if (end == -1) {
          start = 0;
          end = 0;
        }
      }

      String result;
      if (partialResult.length() > 0) {
        if (end > 0) {
          result = partialResult.append(buffer.subSequence(start, end + 1).toString()).toString();
        } else {
          partialResult.setLength(partialResult.length() + end + 1);
          result = partialResult.toString();
        }
      } else {
        try {
          result = buffer.subSequence(start, end + 1).toString();
        } catch (NoSuchMethodError e) {
          result = buffer.toString().substring(start, end + 1);
        }
      }

      buffer.position(buffer.position() + position);
      return result;

    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private Symbol nextSymbol() throws IOException {
    Symbol s;
    if (buffer.position() + position < buffer.limit()) {
      s = symbol(buffer.charAt(position));<commit_msg>Console: fixed jdk6 problem in more elegant way (thanks to Andrey's suggestion)<commit_after>package com.orientechnologies.common.console;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.CharBuffer;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

/**
 * @author <a href='mailto:<EMAIL>'> <NAME> </a>
 */
public class ODFACommandStream implements OCommandStream {
  public static final int      BUFFER_SIZE = 1024;

  private Reader               reader;
  private CharBuffer           buffer;
  private final Set<Character> separators  = new HashSet<Character>(Arrays.asList(';', '\n'));
  private int                  position;
  private int                  start;
  private int                  end;
  private StringBuilder        partialResult;
  private State                state;

  public ODFACommandStream(String commands) {
    reader = new StringReader(commands);
    init();
  }

  public ODFACommandStream(File file) throws FileNotFoundException {
    reader = new BufferedReader(new FileReader(file));
    init();
  }

  private void init() {
    buffer = CharBuffer.allocate(BUFFER_SIZE);
    buffer.flip();
  }

  @Override
  public boolean hasNext() {
    try {
      fillBuffer();

      return buffer.hasRemaining();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private void fillBuffer() throws IOException {
    if (!buffer.hasRemaining()) {
      buffer.clear();
      reader.read(buffer);
      buffer.flip();
    }
  }

  @Override
  public String nextCommand() {
    try {
      fillBuffer();

      partialResult = new StringBuilder();
      state = State.S;
      start = 0;
      end = -1;
      position = 0;
      Symbol s = null;
      while (state != State.E) {
        s = nextSymbol();

        final State newState = transition(state, s);

        if (state == State.S && newState != State.S)
          start = position;

        if (newState == State.A)
          end = position;

        if (newState == State.F)
          throw new IllegalStateException("Unexpected end of file");

        state = newState;
        position++;
      }

      if (s == Symbol.EOF) {
        position--;
        if (end == -1) {
          start = 0;
          end = 0;
        }
      }

      String result;
      if (partialResult.length() > 0) {
        if (end > 0) {
          result = partialResult.append(buffer.subSequence(start, end + 1).toString()).toString();
        } else {
          partialResult.setLength(partialResult.length() + end + 1);
          result = partialResult.toString();
        }
      } else {
        // DON'T PUT THIS ON ONE LINE ONLY BECAUSE WITH JDK6 subSequence() RETURNS A CHAR CharSequence while JDK7+ RETURNS
        // CharBuffer
        final CharSequence cs = buffer.subSequence(start, end + 1);
        result = cs.toString();
      }

      buffer.position(buffer.position() + position);
      return result;

    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private Symbol nextSymbol() throws IOException {
    Symbol s;
    if (buffer.position() + position < buffer.limit()) {
      s = symbol(buffer.charAt(position));
{'fraction_non_alphanumeric': 0.1150509815857556, 'fraction_numerical': 0.005174250494597474, 'mean_word_length': 2.259920634920635, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5566704', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1834, 'n_words': 493}