<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package opennlp.tools.postag;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import opennlp.model.Event;
import opennlp.tools.util.AbstractEventStream;

/**
 * This class reads the {@link POSSample}s from the given {@link Iterator}
 * and converts the {@link POSSample}s into {@link Event}s which
 * can be used by the maxent library for training.
 */
public class POSSampleEventStream extends AbstractEventStream<POSSample> {

  /**
   * The {@link POSContextGenerator} used
   * to create the training {@link Event}s.
   */
  private POSContextGenerator cg;

  /**
   * Initializes the current instance with the given samples and the
   * given {@link POSContextGenerator}.
   *
   * @param samples
   * @param cg
   */
  public POSSampleEventStream(Iterator<POSSample> samples, POSContextGenerator cg) {
    super(samples);

    this.cg = cg;
  }

  /**
   * Initializes the current instance with given samples
   * and a {@link DefaultPOSContextGenerator}.
   * @param samples
   */
  public POSSampleEventStream(Iterator<POSSample> samples) {
    this(samples, new DefaultPOSContextGenerator(null));
  }

  @Override
  protected Iterator<Event> createEvents(POSSample sample) {

    String sentence[] = sample.getSentence();
    String tags[] = sample.getTags();

    List<Event> events = new ArrayList<Event>(sentence.length);

    for (int i=0; i < sentence.length; i++) {

      // it is safe to pass the tags as previous tags because
      // the context generator does not look for non predicted tags
      String[] context = cg.getContext(i, sentence, tags, null);

      events.add(new Event(tags[i], context));
    }

    return events.iterator();
  }
}<commit_msg>updated to allow the sequence iterator to be called multiple times.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package opennlp.tools.postag;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import opennlp.model.Event;
import opennlp.tools.util.AbstractEventStream;

/**
 * This class reads the {@link POSSample}s from the given {@link Iterator}
 * and converts the {@link POSSample}s into {@link Event}s which
 * can be used by the maxent library for training.
 */
public class POSSampleEventStream extends AbstractEventStream<POSSample> {

  /**
   * The {@link POSContextGenerator} used
   * to create the training {@link Event}s.
   */
  private POSContextGenerator cg;

  /**
   * Initializes the current instance with the given samples and the
   * given {@link POSContextGenerator}.
   *
   * @param samples
   * @param cg
   */
  public POSSampleEventStream(Iterator<POSSample> samples, POSContextGenerator cg) {
    super(samples);

    this.cg = cg;
  }

  /**
   * Initializes the current instance with given samples
   * and a {@link DefaultPOSContextGenerator}.
   * @param samples
   */
  public POSSampleEventStream(Iterator<POSSample> samples) {
    this(samples, new DefaultPOSContextGenerator(null));
  }

  @Override
  protected Iterator<Event> createEvents(POSSample sample) {
    String sentence[] = sample.getSentence();
    String tags[] = sample.getTags();
    List<Event> events = generateEvents(sentence,tags,cg);
    return events.iterator();
  }
  
  public static List<Event> generateEvents(String[] sentence, String[] tags, POSContextGenerator cg){
    List<Event> events = new ArrayList<Event>(sentence.length);

    for (int i=0; i < sentence.length; i++) {

      // it is safe to pass the tags as previous tags because
      // the context generator does not look for non predicted tags
      String[] context = cg.getContext(i, sentence, tags, null);

      events.add(new Event(tags[i], context));
    }
    return events;
  }
}
{'fraction_non_alphanumeric': 0.09101208459214502, 'fraction_numerical': 0.0018882175226586104, 'mean_word_length': 3.7634892086330933, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5215622', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1411, 'n_words': 562}