// Path: src/main/java/net/java/sen/StringTagger.java
// public class StringTagger {
//   
//   /**
//    * The Viterbi analyser used to decompose strings
//    */
//   private final Viterbi viterbi;
//   
//   /**
//    * {@link StreamFilter}s to apply during analysis
//    */
//   private List<StreamFilter> filterList = new ArrayList<StreamFilter>();
//   
//   /**
//    * Apply the pre-processing phase of all attached {@link StreamFilter}s to
//    * the input sentence
//    *
//    * @param sentence The input sentence
//    */
//   private void filterPreProcess(Sentence sentence) {
//     for (int i = 0; i < filterList.size(); i++) {
//       filterList.get(i).preProcess(sentence);
//     }
//   }
//   
//   /**
//    * Apply the post-processing phase of all attached {@link StreamFilter}s to
//    * the analysed {@link Token}s
//    *
//    * @param tokens The analysed {@link Token}s
//    * @return The filtered {@link Token}s
//    */
//   private List<Token> filterPostProcess(List<Token> tokens) {
//     for (int i = filterList.size() - 1; i >= 0; i--) {
//       StreamFilter filter = filterList.get(i);
//       tokens = filter.postProcess(tokens);
//     }
//     
//     return tokens;
//   }
//   
//   /**
//    * Add a {@link StreamFilter} to be applied during analysis
//    *
//    * @param filter The {@link StreamFilter} to add
//    */
//   public void addFilter(StreamFilter filter) {
//     filterList.add(filter);
//   }
//   
//   /**
//    * Remove all current {@link StreamFilter}s
//    */
//   public void removeFilters() {
//     filterList.clear();
//   }
//   
//   /**
//    * Decompose a string into its most likely constituent morphemes
//    * 
//    * @param surface The string to analyse
//    * @return An array of {@link Token}s representing the most likely morphemes
//    * @throws IOException 
//    */
//   public List<Token> analyze(String surface, List<Token> reuse) throws IOException {
//     Sentence sentence = new Sentence(surface.toCharArray());
//     filterPreProcess(sentence);
//     
//     List<Token> tokens = viterbi.getBestTokens(sentence, reuse);
//     
//     tokens = filterPostProcess(tokens);
//     
//     return tokens;
//   }
//   
//   /**
//    * @deprecated use {@link #analyze(String, List)} instead.
//    */
//   @Deprecated
//   public List<Token> analyze(String surface) throws IOException {
//     return analyze(surface, new ArrayList<Token>());
//   }
//   
//   /**
//    * Decompose a string into its most likely constituent morphemes
//    * 
//    * @param surface The string to analyse
//    * @return An array of {@link Token}s representing the most likely morphemes
//    * @throws IOException 
//    */
//   public List<Token> analyze(char[] surface, List<Token> reuse) throws IOException {
//     Sentence sentence = new Sentence(surface);
//     filterPreProcess(sentence);
//     
//     List<Token> tokens = viterbi.getBestTokens(sentence, reuse);
//     
//     tokens = filterPostProcess(tokens);
//     
//     return tokens;
//   }
//   
//   /**
//    * @deprecated use {@link #analyze(char[], List)} instead.
//    */
//   @Deprecated
//   public List<Token> analyze(char[] surface) throws IOException {
//     return analyze(surface, new ArrayList<Token>());
//   }
//   
//   /**
//    * @param tokenizer The Tokenizer to use for analysis 
//    */
//   public StringTagger(Tokenizer tokenizer) {
//     this.viterbi = new Viterbi(tokenizer);
//   }
// }


import java.io.IOException;
import java.io.Reader;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import net.java.sen.StringTagger;
import net.java.sen.dictionary.Token;
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.analysis.gosen;




/**
 * Breaks text into sentences according to UAX #29: Unicode Text Segmentation
 * (http://www.unicode.org/reports/tr29/)
 * <p>
 */
public final class StreamTagger2 {
  private static final int IOBUFFER = 4096;
  private final char buffer[] = new char[IOBUFFER];
  /** true length of text in the buffer */
  private int length = 0; 
  /** length in buffer that can be evaluated safely, up to a safe end point */
  private int usableLength = 0; 
  /** accumulated offset of previous buffers for this reader, for offsetAtt */
  private int offset = 0;
  
  private StringTagger tagger;
  private Reader input;
  private final BreakIterator breaker = BreakIterator.getSentenceInstance(Locale.JAPANESE); /* tokenizes a char[] of text */
  private final CharArrayIterator iterator = new CharArrayIterator();
  private List<Token> tokens = new ArrayList<Token>();
  private int index = 0;

  /**
   * Construct a new StreamTagger2 that breaks text into words from the given Reader.
   */
  public StreamTagger2(StringTagger tagger, Reader input) {
    this.tagger = tagger;
    this.input = input;
  }

  public Token next() throws IOException {
    if (tokens == null || index >= tokens.size()) {
// Path: src/main/java/net/java/sen/StringTagger.java
// public class StringTagger {
//   
//   /**
//    * The Viterbi analyser used to decompose strings
//    */
//   private final Viterbi viterbi;
//   
//   /**
//    * {@link StreamFilter}s to apply during analysis
//    */
//   private List<StreamFilter> filterList = new ArrayList<StreamFilter>();
//   
//   /**
//    * Apply the pre-processing phase of all attached {@link StreamFilter}s to
//    * the input sentence
//    *
//    * @param sentence The input sentence
//    */
//   private void filterPreProcess(Sentence sentence) {
//     for (int i = 0; i < filterList.size(); i++) {
//       filterList.get(i).preProcess(sentence);
//     }
//   }
//   
//   /**
//    * Apply the post-processing phase of all attached {@link StreamFilter}s to
//    * the analysed {@link Token}s
//    *
//    * @param tokens The analysed {@link Token}s
//    * @return The filtered {@link Token}s
//    */
//   private List<Token> filterPostProcess(List<Token> tokens) {
//     for (int i = filterList.size() - 1; i >= 0; i--) {
//       StreamFilter filter = filterList.get(i);
//       tokens = filter.postProcess(tokens);
//     }
//     
//     return tokens;
//   }
//   
//   /**
//    * Add a {@link StreamFilter} to be applied during analysis
//    *
//    * @param filter The {@link StreamFilter} to add
//    */
//   public void addFilter(StreamFilter filter) {
//     filterList.add(filter);
//   }
//   
//   /**
//    * Remove all current {@link StreamFilter}s
//    */
//   public void removeFilters() {
//     filterList.clear();
//   }
//   
//   /**
//    * Decompose a string into its most likely constituent morphemes
//    * 
//    * @param surface The string to analyse
//    * @return An array of {@link Token}s representing the most likely morphemes
//    * @throws IOException 
//    */
//   public List<Token> analyze(String surface, List<Token> reuse) throws IOException {
//     Sentence sentence = new Sentence(surface.toCharArray());
//     filterPreProcess(sentence);
//     
//     List<Token> tokens = viterbi.getBestTokens(sentence, reuse);
//     
//     tokens = filterPostProcess(tokens);
//     
//     return tokens;
//   }
//   
//   /**
//    * @deprecated use {@link #analyze(String, List)} instead.
//    */
//   @Deprecated
//   public List<Token> analyze(String surface) throws IOException {
//     return analyze(surface, new ArrayList<Token>());
//   }
//   
//   /**
//    * Decompose a string into its most likely constituent morphemes
//    * 
//    * @param surface The string to analyse
//    * @return An array of {@link Token}s representing the most likely morphemes
//    * @throws IOException 
//    */
//   public List<Token> analyze(char[] surface, List<Token> reuse) throws IOException {
//     Sentence sentence = new Sentence(surface);
//     filterPreProcess(sentence);
//     
//     List<Token> tokens = viterbi.getBestTokens(sentence, reuse);
//     
//     tokens = filterPostProcess(tokens);
//     
//     return tokens;
//   }
//   
//   /**
//    * @deprecated use {@link #analyze(char[], List)} instead.
//    */
//   @Deprecated
//   public List<Token> analyze(char[] surface) throws IOException {
//     return analyze(surface, new ArrayList<Token>());
//   }
//   
//   /**
//    * @param tokenizer The Tokenizer to use for analysis 
//    */
//   public StringTagger(Tokenizer tokenizer) {
//     this.viterbi = new Viterbi(tokenizer);
//   }
// }


// Path: src/main/java/org/apache/lucene/analysis/gosen/StreamTagger2.java
import java.io.IOException;
import java.io.Reader;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import net.java.sen.StringTagger;
import net.java.sen.dictionary.Token;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.analysis.gosen;




/**
 * Breaks text into sentences according to UAX #29: Unicode Text Segmentation
 * (http://www.unicode.org/reports/tr29/)
 * <p>
 */
public final class StreamTagger2 {
  private static final int IOBUFFER = 4096;
  private final char buffer[] = new char[IOBUFFER];
  /** true length of text in the buffer */
  private int length = 0; 
  /** length in buffer that can be evaluated safely, up to a safe end point */
  private int usableLength = 0; 
  /** accumulated offset of previous buffers for this reader, for offsetAtt */
  private int offset = 0;
  
  private StringTagger tagger;
  private Reader input;
  private final BreakIterator breaker = BreakIterator.getSentenceInstance(Locale.JAPANESE); /* tokenizes a char[] of text */
  private final CharArrayIterator iterator = new CharArrayIterator();
  private List<Token> tokens = new ArrayList<Token>();
  private int index = 0;

  /**
   * Construct a new StreamTagger2 that breaks text into words from the given Reader.
   */
  public StreamTagger2(StringTagger tagger, Reader input) {
    this.tagger = tagger;
    this.input = input;
  }

  public Token next() throws IOException {
    if (tokens == null || index >= tokens.size()) {
      if (length == 0)