<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package opennlp.tools.namefind;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.EventModelSequenceTrainer;
import opennlp.tools.ml.EventTrainer;
import opennlp.tools.ml.SequenceTrainer;
import opennlp.tools.ml.TrainerFactory;
import opennlp.tools.ml.TrainerFactory.TrainerType;
import opennlp.tools.ml.model.Event;
import opennlp.tools.ml.model.MaxentModel;
import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Sequence;
import opennlp.tools.util.SequenceCodec;
import opennlp.tools.util.SequenceValidator;
import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
import opennlp.tools.util.featuregen.AdditionalContextFeatureGenerator;
import opennlp.tools.util.featuregen.BigramNameFeatureGenerator;
import opennlp.tools.util.featuregen.CachedFeatureGenerator;
import opennlp.tools.util.featuregen.FeatureGeneratorResourceProvider;
import opennlp.tools.util.featuregen.GeneratorFactory;
import opennlp.tools.util.featuregen.OutcomePriorFeatureGenerator;
import opennlp.tools.util.featuregen.PreviousMapFeatureGenerator;
import opennlp.tools.util.featuregen.SentenceFeatureGenerator;
import opennlp.tools.util.featuregen.TokenClassFeatureGenerator;
import opennlp.tools.util.featuregen.TokenFeatureGenerator;
import opennlp.tools.util.featuregen.WindowFeatureGenerator;
import opennlp.tools.util.model.ModelUtil;

/**
 * Class for creating a maximum-entropy-based name finder.
 */
public class NameFinderME implements TokenNameFinder {

  private static String[][] EMPTY = new String[0][0];
  public static final int DEFAULT_BEAM_SIZE = 3;
  private static final Pattern typedOutcomePattern = Pattern.compile("(.+)-\\w+");

  public static final String START = "start";
  public static final String CONTINUE = "cont";
  public static final String OTHER = "other";

  private SequenceCodec<String> seqCodec = new BioCodec();

  protected SequenceClassificationModel<String> model;

  protected NameContextGenerator contextGenerator;
  private Sequence bestSequence;

  private AdditionalContextFeatureGenerator additionalContextFeatureGenerator
          = new AdditionalContextFeatureGenerator();
  private SequenceValidator<String> sequenceValidator;

  public NameFinderME(TokenNameFinderModel model) {

    TokenNameFinderFactory factory = model.getFactory();

    seqCodec = factory.createSequenceCodec();
    sequenceValidator = seqCodec.createSequenceValidator();
    this.model = model.getNameFinderSequenceModel();
    contextGenerator = factory.createContextGenerator();

    // TODO: We should deprecate this. And come up with a better solution!
    contextGenerator.addFeatureGenerator(
            new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));
  }

  /**
   * Initializes the name finder with the specified model.
   *
   * @param model
   * @param beamSize
   *
   * @deprecated the beam size is now configured during training time in the
   * trainer parameter file via beamSearch.beamSize
   *
   * @deprecated Use {@link #NameFinderME(TokenNameFinderModel)} instead and use
   * the {@link TokenNameFinderFactory} to configure it.
   */
  @Deprecated
  public NameFinderME(TokenNameFinderModel model, AdaptiveFeatureGenerator generator, int beamSize,
          SequenceValidator<String> sequenceValidator) {

    seqCodec = model.getFactory().createSequenceCodec();

    this.sequenceValidator = sequenceValidator;

    // TODO: getNameFinderModel should be removed! Instead the model should always return
    // a sequence classification model
    // To maintain backward compatibility this should be done later, e.g. for 1.7.0
    if (model.getNameFinderSequenceModel() != null) {
      this.model = model.getNameFinderSequenceModel();
    } else {
      this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
              model.getNameFinderModel());
    }

    // If generator is provided always use that one
    if (generator != null) {
      contextGenerator = new DefaultNameContextGenerator(generator);
    } else {
      // If model has a generator use that one, otherwise create default
      AdaptiveFeatureGenerator featureGenerator = model.createFeatureGenerators();

      if (featureGenerator == null) {
        featureGenerator = createFeatureGenerator();
      }

      contextGenerator = new DefaultNameContextGenerator(featureGenerator);
    }

    // NOTE: This didn't turn out to work well ... anybody using this actually ?!
    contextGenerator.addFeatureGenerator(
            new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));

    if (this.sequenceValidator == null) {
      this.sequenceValidator = new NameFinderSequenceValidator();
    }
  }

  /**
   * @deprecated the beam size is now configured during training time in the
   * trainer parameter file via beamSearch.beamSize
   */
  @Deprecated
  public NameFinderME(TokenNameFinderModel model, AdaptiveFeatureGenerator generator, int beamSize) {
    this(model, generator, beamSize, null);
  }

  /**
   * @deprecated the beam size is now configured during training time in the
   * trainer parameter file via beamSearch.beamSize
   */
  @Deprecated
  public NameFinderME(TokenNameFinderModel model, int beamSize) {
    this(model, null, beamSize);
  }

  static AdaptiveFeatureGenerator createFeatureGenerator() {
    return new CachedFeatureGenerator(
            new AdaptiveFeatureGenerator[]{
              new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
              new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
              new OutcomePriorFeatureGenerator(),
              new PreviousMapFeatureGenerator(),
              new BigramNameFeatureGenerator(),
              new SentenceFeatureGenerator(true, false)
            });
  }

  private static AdaptiveFeatureGenerator createFeatureGenerator(
          byte[] generatorDescriptor, final Map<String, Object> resources)
          throws IOException {
    AdaptiveFeatureGenerator featureGenerator;

    if (generatorDescriptor != null) {
      featureGenerator = GeneratorFactory.create(new ByteArrayInputStream(
              generatorDescriptor), new FeatureGeneratorResourceProvider() {<commit_msg>OPENNLP-822 Deprecated createFeatureGenerator and added a comment.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package opennlp.tools.namefind;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.EventModelSequenceTrainer;
import opennlp.tools.ml.EventTrainer;
import opennlp.tools.ml.SequenceTrainer;
import opennlp.tools.ml.TrainerFactory;
import opennlp.tools.ml.TrainerFactory.TrainerType;
import opennlp.tools.ml.model.Event;
import opennlp.tools.ml.model.MaxentModel;
import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Sequence;
import opennlp.tools.util.SequenceCodec;
import opennlp.tools.util.SequenceValidator;
import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
import opennlp.tools.util.featuregen.AdditionalContextFeatureGenerator;
import opennlp.tools.util.featuregen.BigramNameFeatureGenerator;
import opennlp.tools.util.featuregen.CachedFeatureGenerator;
import opennlp.tools.util.featuregen.FeatureGeneratorResourceProvider;
import opennlp.tools.util.featuregen.GeneratorFactory;
import opennlp.tools.util.featuregen.OutcomePriorFeatureGenerator;
import opennlp.tools.util.featuregen.PreviousMapFeatureGenerator;
import opennlp.tools.util.featuregen.SentenceFeatureGenerator;
import opennlp.tools.util.featuregen.TokenClassFeatureGenerator;
import opennlp.tools.util.featuregen.TokenFeatureGenerator;
import opennlp.tools.util.featuregen.WindowFeatureGenerator;
import opennlp.tools.util.model.ModelUtil;

/**
 * Class for creating a maximum-entropy-based name finder.
 */
public class NameFinderME implements TokenNameFinder {

  private static String[][] EMPTY = new String[0][0];
  public static final int DEFAULT_BEAM_SIZE = 3;
  private static final Pattern typedOutcomePattern = Pattern.compile("(.+)-\\w+");

  public static final String START = "start";
  public static final String CONTINUE = "cont";
  public static final String OTHER = "other";

  private SequenceCodec<String> seqCodec = new BioCodec();

  protected SequenceClassificationModel<String> model;

  protected NameContextGenerator contextGenerator;
  private Sequence bestSequence;

  private AdditionalContextFeatureGenerator additionalContextFeatureGenerator
          = new AdditionalContextFeatureGenerator();
  private SequenceValidator<String> sequenceValidator;

  public NameFinderME(TokenNameFinderModel model) {

    TokenNameFinderFactory factory = model.getFactory();

    seqCodec = factory.createSequenceCodec();
    sequenceValidator = seqCodec.createSequenceValidator();
    this.model = model.getNameFinderSequenceModel();
    contextGenerator = factory.createContextGenerator();

    // TODO: We should deprecate this. And come up with a better solution!
    contextGenerator.addFeatureGenerator(
            new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));
  }

  /**
   * Initializes the name finder with the specified model.
   *
   * @param model
   * @param beamSize
   *
   * @deprecated the beam size is now configured during training time in the
   * trainer parameter file via beamSearch.beamSize
   *
   * @deprecated Use {@link #NameFinderME(TokenNameFinderModel)} instead and use
   * the {@link TokenNameFinderFactory} to configure it.
   */
  @Deprecated
  public NameFinderME(TokenNameFinderModel model, AdaptiveFeatureGenerator generator, int beamSize,
          SequenceValidator<String> sequenceValidator) {

    seqCodec = model.getFactory().createSequenceCodec();

    this.sequenceValidator = sequenceValidator;

    // TODO: getNameFinderModel should be removed! Instead the model should always return
    // a sequence classification model
    // To maintain backward compatibility this should be done later, e.g. for 1.7.0
    if (model.getNameFinderSequenceModel() != null) {
      this.model = model.getNameFinderSequenceModel();
    } else {
      this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
              model.getNameFinderModel());
    }

    // If generator is provided always use that one
    if (generator != null) {
      contextGenerator = new DefaultNameContextGenerator(generator);
    } else {
      // If model has a generator use that one, otherwise create default
      AdaptiveFeatureGenerator featureGenerator = model.createFeatureGenerators();

      if (featureGenerator == null) {
        featureGenerator = createFeatureGenerator();
      }

      contextGenerator = new DefaultNameContextGenerator(featureGenerator);
    }

    // NOTE: This didn't turn out to work well ... anybody using this actually ?!
    contextGenerator.addFeatureGenerator(
            new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));

    if (this.sequenceValidator == null) {
      this.sequenceValidator = new NameFinderSequenceValidator();
    }
  }

  /**
   * @deprecated the beam size is now configured during training time in the
   * trainer parameter file via beamSearch.beamSize
   */
  @Deprecated
  public NameFinderME(TokenNameFinderModel model, AdaptiveFeatureGenerator generator, int beamSize) {
    this(model, generator, beamSize, null);
  }

  /**
   * @deprecated the beam size is now configured during training time in the
   * trainer parameter file via beamSearch.beamSize
   */
  @Deprecated
  public NameFinderME(TokenNameFinderModel model, int beamSize) {
    this(model, null, beamSize);
  }

  @Deprecated
  /**
   * @deprecated the default feature generation is now always included in the models and loaded
   * if not by the factory. Subclasses using this methods should do the same.
   */
  static AdaptiveFeatureGenerator createFeatureGenerator() {
    return new CachedFeatureGenerator(
            new AdaptiveFeatureGenerator[]{
              new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
              new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
              new OutcomePriorFeatureGenerator(),
              new PreviousMapFeatureGenerator(),
              new BigramNameFeatureGenerator(),
              new SentenceFeatureGenerator(true, false)
            });
  }

  private static AdaptiveFeatureGenerator createFeatureGenerator(
          byte[] generatorDescriptor, final Map<String, Object> resources)
          throws IOException {
    AdaptiveFeatureGenerator featureGenerator;

    if (generatorDescriptor != null) {
      featureGenerator = GeneratorFactory.create(new ByteArrayInputStream(
              generatorDescriptor), new FeatureGeneratorResourceProvider() {
{'fraction_non_alphanumeric': 0.07856244622410484, 'fraction_numerical': 0.002581242967767556, 'mean_word_length': 4.301754385964912, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4230856', 'n_tokens_mistral': 3900, 'n_tokens_neox': 3826, 'n_words': 1134}