<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.tools.eval;

import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.junit.Assert;
import org.junit.Test;

import opennlp.tools.chunker.Chunker;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.formats.LeipzigDoccatSampleStream;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinder;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.Parser;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTagger;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.sentdetect.SentenceDetector;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;

/**
 * This tests ensures that the existing SourceForge models perform
 * like they are expected to.
 * <p>
 * To run this tests external the leipzig sentences files is needed:
 * leipzig/eng_news_2010_300K-sentences.txt, this file can be
 * obtained from the leipzig corpus project. <br>
 * <p>
 * And all the SourceForge models:<br>
 * - models-sf/en-sent.bin<br>
 * - models-sf/en-token.bin<br>
 * - models-sf/en-ner-date.bin<br>
 * - models-sf/en-ner-location.binn<br>
 * - models-sf/en-ner-money.bin<br>
 * - models-sf/en-ner-organization.bin<br>
 * - models-sf/en-ner-percentage.bi<br>
 * - models-sf/en-ner-person.bin<br>
 * - models-sf/en-ner-time.bin<br>
 * - models-sf/en-chunker.bin<br>
 * - models-sf/en-pos-maxent.bin<br>
 * - models-sf/en-pos-perceptron.bin<br>
 * - models-sf/en-parser-chunking.bin.bin<br>
 */
public class SourceForgeModelEval {

  private static MessageDigest createDigest() {
    try {
      return MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IllegalStateException(e);
    }
  }

  @Test
  public void ensureTestDataIsCorrect() throws IOException {
    MessageDigest digest = createDigest();

    try (ObjectStream<String> lines = new PlainTextByLineStream(
        new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
            "leipzig/eng_news_2010_300K-sentences.txt")), Charset.forName("UTF-8"))) {

      String line;
      while ((line = lines.read()) != null) {
        digest.update(line.getBytes("UTF-8"));
      }

      Assert.assertEquals(new BigInteger("248567841356936801447294643695012852392"),
          new BigInteger(1, digest.digest()));
    }
  }

  @Test
  public void evalSentenceModel() throws IOException {

    SentenceModel model = new SentenceModel(
        new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-sent.bin"));

    MessageDigest digest = createDigest();

    SentenceDetector sentenceDetector = new SentenceDetectorME(model);

    StringBuilder text = new StringBuilder();

    try (ObjectStream<DocumentSample> lineBatches = new LeipzigDoccatSampleStream("en", 25,
        new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
            "leipzig/eng_news_2010_300K-sentences.txt")))) {<commit_msg>OPENNLP-1041: Verify test data before running tests<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.tools.eval;

import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

import opennlp.tools.chunker.Chunker;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.formats.LeipzigDoccatSampleStream;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinder;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.Parser;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTagger;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.sentdetect.SentenceDetector;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;

/**
 * This tests ensures that the existing SourceForge models perform
 * like they are expected to.
 * <p>
 * To run this tests external the leipzig sentences files is needed:
 * leipzig/eng_news_2010_300K-sentences.txt, this file can be
 * obtained from the leipzig corpus project. <br>
 * <p>
 * And all the SourceForge models:<br>
 * - models-sf/en-sent.bin<br>
 * - models-sf/en-token.bin<br>
 * - models-sf/en-ner-date.bin<br>
 * - models-sf/en-ner-location.binn<br>
 * - models-sf/en-ner-money.bin<br>
 * - models-sf/en-ner-organization.bin<br>
 * - models-sf/en-ner-percentage.bi<br>
 * - models-sf/en-ner-person.bin<br>
 * - models-sf/en-ner-time.bin<br>
 * - models-sf/en-chunker.bin<br>
 * - models-sf/en-pos-maxent.bin<br>
 * - models-sf/en-pos-perceptron.bin<br>
 * - models-sf/en-parser-chunking.bin.bin<br>
 */
public class SourceForgeModelEval {

  private static MessageDigest createDigest() {
    try {
      return MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IllegalStateException(e);
    }
  }

  @BeforeClass
  public static void ensureTestDataIsCorrect() throws IOException {
    MessageDigest digest = createDigest();

    try (ObjectStream<String> lines = new PlainTextByLineStream(
        new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
            "leipzig/eng_news_2010_300K-sentences.txt")), Charset.forName("UTF-8"))) {

      String line;
      while ((line = lines.read()) != null) {
        digest.update(line.getBytes("UTF-8"));
      }

      Assert.assertEquals(new BigInteger("248567841356936801447294643695012852392"),
          new BigInteger(1, digest.digest()));
    }
  }

  @Test
  public void evalSentenceModel() throws IOException {

    SentenceModel model = new SentenceModel(
        new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-sent.bin"));

    MessageDigest digest = createDigest();

    SentenceDetector sentenceDetector = new SentenceDetectorME(model);

    StringBuilder text = new StringBuilder();

    try (ObjectStream<DocumentSample> lineBatches = new LeipzigDoccatSampleStream("en", 25,
        new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
            "leipzig/eng_news_2010_300K-sentences.txt")))) {
{'fraction_non_alphanumeric': 0.10563228312473026, 'fraction_numerical': 0.015537332757876565, 'mean_word_length': 4.461991750147319, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6347800', 'n_tokens_mistral': 3010, 'n_tokens_neox': 2878, 'n_words': 659}