<commit_before>/* JLanguageTool, a natural language style checker 
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package de.danielnaber.languagetool.tokenizers;

import java.util.Iterator;
import java.util.Vector;

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;

/**
 * @author <NAME>
 */
public class SentenceTokenizerTest extends TestCase {

  public SentenceTokenizerTest(String arg0) {
    super(arg0);
  }

  public static Test suite() {
    TestSuite suite = new TestSuite("SentenceTokenizerTest");
    suite.addTestSuite(SentenceTokenizerTest.class);

    return suite;
  }

  public void testTokenize() {
    // WICHTIG: Sätze müssen mit Leerzeichen enden, so dass
    // sie hintereinandergehängt wie ein Text aussehen:
    testSplit(new String[] { "Dies ist ein Satz." });
    testSplit(new String[] { "Dies ist ein Satz. ", "Noch einer." });
    testSplit(new String[] { "Ein Satz! ", "Noch einer." });
    testSplit(new String[] { "Ein Satz... ", "Noch einer." });

    testSplit(new String[] { "Heute ist der 13.12.2004." });
    testSplit(new String[] { "Heute ist der 13. Dezember." });
    testSplit(new String[] { "Heute ist der 1. Januar." });

    // Tests taken from LanguageTool's SentenceSplitterTest.py:
    testSplit(new String[] { "This is a sentence. " });
    testSplit(new String[] { "This is a sentence. ", "And this is another one." });
    testSplit(new String[] { "This is a sentence.", "Isn't it?", "Yes, it is." });
    testSplit(new String[] { "This is e.g. Mr. Smith, who talks slowly...",
        "But this is another sentence." });
    testSplit(new String[] { "Chanel no. 5 is blah." });
    testSplit(new String[] { "Mrs. Jones gave Peter $4.5, to buy Chanel No 5.",
        "He never came back." });
    testSplit(new String[] { "On p. 6 there's nothing. ", "Another sentence." });
    testSplit(new String[] { "Leave me alone!, he yelled. ", "Another sentence." });
    testSplit(new String[] { "\"Leave me alone!\", he yelled." });
    testSplit(new String[] { "'Leave me alone!', he yelled. ", "Another sentence." });
    testSplit(new String[] { "'Leave me alone!,' he yelled. ", "Another sentence." });
    testSplit(new String[] { "This works on the phrase level, i.e. not on the word level." });
    testSplit(new String[] { "Let's meet at 5 p.m. in the main street." });
    testSplit(new String[] { "James comes from the U.K. where he worked as a programmer." });
    testSplit(new String[] { "Don't split strings like U.S.A. please." });
    testSplit(new String[] { "Don't split strings like U. S. A. either." });
    testSplit(new String[] { "Don't split... ", "Well you know. ", "Here comes more text." });
    testSplit(new String[] { "Don't split... well you know. ", "Here comes more text." });
    testSplit(new String[] { "The \".\" should not be a delimiter in quotes." });
    testSplit(new String[] { "\"Here he comes!\" she said." });
    testSplit(new String[] { "\"Here he comes!\", she said." });
    testSplit(new String[] { "\"Here he comes.\" ", "But this is another sentence." });
    testSplit(new String[] { "\"Here he comes!\". ", "That's what he said." });
    testSplit(new String[] { "The sentence ends here. ", "(Another sentence.)" });
    // known to fail:
    // testSplit(new String[]{"He won't. ", "Really."});
    testSplit(new String[] { "He won't go. ", "Really." });
    testSplit(new String[] { "He won't say no.", "Not really." });
    testSplit(new String[] { "He won't say No.", "Not really." });
    testSplit(new String[] { "He won't say no. 5 is better. ", "Not really." });
    testSplit(new String[] { "He won't say No. 5 is better. ", "Not really." });
    testSplit(new String[] { "They met at 5 p.m. on Thursday." });
    testSplit(new String[] { "They met at 5 p.m. ", "It was Thursday." });
    testSplit(new String[] { "This is it: a test." });
    // two returns = paragraph = new sentence:
    testSplit(new String[] { "He won't\n\n", "Really." });
    // Missing space after sentence end:
    testSplit(new String[] { "James is from the Ireland!", "He lives in Spain now." });
    // From the abbreviation list:
    testSplit(new String[] { "<NAME>. have built a succesful company." });
  }

  private void testSplit(String[] sentences) {
    StringBuffer inputString = new StringBuffer();
    Vector input = new Vector();
    for (int i = 0; i < sentences.length; i++) {
      input.add(sentences[i]);
    }
    for (Iterator iter = input.iterator(); iter.hasNext();) {
      String s = (String) iter.next();
      inputString.append(s);
    }
    SentenceTokenizer stokenizer = new SentenceTokenizer();
    assertEquals(input, stokenizer.tokenize(inputString.toString()));
  }
}<commit_msg>small cleanup; two new tests<commit_after>/* JLanguageTool, a natural language style checker 
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package de.danielnaber.languagetool.tokenizers;

import java.util.Iterator;
import java.util.Vector;

import junit.framework.TestCase;

/**
 * @author <NAME>
 */
public class SentenceTokenizerTest extends TestCase {

  public void testTokenize() {
    // NOTE: sentences here need to end with a space character so they
    // have correct whitespace when appended:
    testSplit(new String[] { "Dies ist ein Satz." });
    testSplit(new String[] { "Dies ist ein Satz. ", "Noch einer." });
    testSplit(new String[] { "Ein Satz! ", "Noch einer." });
    testSplit(new String[] { "Ein Satz... ", "Noch einer." });

    testSplit(new String[] { "Heute ist der 13.12.2004." });
    testSplit(new String[] { "Heute ist der 13. Dezember." });
    testSplit(new String[] { "Heute ist der 1. Januar." });
    
    // incomplete sentences, need to work for on-thy-fly checking of texts:
    testSplit(new String[] { "Here's a" });
    testSplit(new String[] { "Here's a sentence. ", "And here's one that's not comp" });

    // Tests taken from LanguageTool's SentenceSplitterTest.py:
    testSplit(new String[] { "This is a sentence. " });
    testSplit(new String[] { "This is a sentence. ", "And this is another one." });
    testSplit(new String[] { "This is a sentence.", "Isn't it?", "Yes, it is." });
    testSplit(new String[] { "This is e.g. Mr. Smith, who talks slowly...",
        "But this is another sentence." });
    testSplit(new String[] { "Chanel no. 5 is blah." });
    testSplit(new String[] { "Mrs. Jones gave Peter $4.5, to buy Chanel No 5.",
        "He never came back." });
    testSplit(new String[] { "On p. 6 there's nothing. ", "Another sentence." });
    testSplit(new String[] { "Leave me alone!, he yelled. ", "Another sentence." });
    testSplit(new String[] { "\"Leave me alone!\", he yelled." });
    testSplit(new String[] { "'Leave me alone!', he yelled. ", "Another sentence." });
    testSplit(new String[] { "'Leave me alone!,' he yelled. ", "Another sentence." });
    testSplit(new String[] { "This works on the phrase level, i.e. not on the word level." });
    testSplit(new String[] { "Let's meet at 5 p.m. in the main street." });
    testSplit(new String[] { "James comes from the U.K. where he worked as a programmer." });
    testSplit(new String[] { "Don't split strings like U.S.A. please." });
    testSplit(new String[] { "Don't split strings like U. S. A. either." });
    testSplit(new String[] { "Don't split... ", "Well you know. ", "Here comes more text." });
    testSplit(new String[] { "Don't split... well you know. ", "Here comes more text." });
    testSplit(new String[] { "The \".\" should not be a delimiter in quotes." });
    testSplit(new String[] { "\"Here he comes!\" she said." });
    testSplit(new String[] { "\"Here he comes!\", she said." });
    testSplit(new String[] { "\"Here he comes.\" ", "But this is another sentence." });
    testSplit(new String[] { "\"Here he comes!\". ", "That's what he said." });
    testSplit(new String[] { "The sentence ends here. ", "(Another sentence.)" });
    // known to fail:
    // testSplit(new String[]{"He won't. ", "Really."});
    testSplit(new String[] { "He won't go. ", "Really." });
    testSplit(new String[] { "He won't say no.", "Not really." });
    testSplit(new String[] { "He won't say No.", "Not really." });
    testSplit(new String[] { "He won't say no. 5 is better. ", "Not really." });
    testSplit(new String[] { "He won't say No. 5 is better. ", "Not really." });
    testSplit(new String[] { "They met at 5 p.m. on Thursday." });
    testSplit(new String[] { "They met at 5 p.m. ", "It was Thursday." });
    testSplit(new String[] { "This is it: a test." });
    // two returns = paragraph = new sentence:
    testSplit(new String[] { "He won't\n\n", "Really." });
    // Missing space after sentence end:
    testSplit(new String[] { "James is from the Ireland!", "He lives in Spain now." });
    // From the abbreviation list:
    testSplit(new String[] { "<NAME>. have built a succesful company." });
  }

  private void testSplit(String[] sentences) {
    StringBuffer inputString = new StringBuffer();
    Vector input = new Vector();
    for (int i = 0; i < sentences.length; i++) {
      input.add(sentences[i]);
    }
    for (Iterator iter = input.iterator(); iter.hasNext();) {
      String s = (String) iter.next();
      inputString.append(s);
    }
    SentenceTokenizer stokenizer = new SentenceTokenizer();
    assertEquals(input, stokenizer.tokenize(inputString.toString()));
  }

}
{'fraction_non_alphanumeric': 0.15627286027798098, 'fraction_numerical': 0.0073152889539136795, 'mean_word_length': 3.260615504479938, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4828672', 'n_tokens_mistral': 3381, 'n_tokens_neox': 3219, 'n_words': 1253}