<commit_before>/* LanguageTool, a natural language style checker
 * Copyright (C) 2015 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.dumpcheck;

import org.languagetool.Language;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Scanner;
import java.util.regex.Pattern;

/**
 * Provides access to the relevant sentences of a plain text file
 * with one sentence per line.
 * @since 3.0
 */
public class PlainTextSentenceSource extends SentenceSource {

  private final List<String> sentences;
  private final Scanner scanner;

  // Each sentence is one article, but count anyway so it's coherent with what the Wikipedia code does:
  private int articleCount = 0;

  public PlainTextSentenceSource(InputStream textInput, Language language) {
    this(textInput, language, null);
  }
  
  /** @since 3.0 */
  public PlainTextSentenceSource(InputStream textInput, Language language, Pattern filter) {
    super(language, filter);
    scanner = new Scanner(textInput);
    sentences = new ArrayList<>();
  }

  @Override
  public boolean hasNext() {
    fillSentences();
    return sentences.size() > 0;
  }

  @Override
  public Sentence next() {
    fillSentences();
    if (sentences.isEmpty()) {
      throw new NoSuchElementException();
    }
    return new Sentence(sentences.remove(0), getSource(), "<plaintext>", null, ++articleCount);
  }

  @Override
  public String getSource() {
    return "plaintext";
  }

  private void fillSentences() {
    while (sentences.isEmpty() && scanner.hasNextLine()) {
      String line = scanner.nextLine();
      if (line.isEmpty()) {
        continue;
      }
      if (acceptSentence(line)) {
        sentences.add(line);
      }
    }
  }
<commit_msg>take source url from simple syntax in plain text file<commit_after>/* LanguageTool, a natural language style checker
 * Copyright (C) 2015 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.dumpcheck;

import org.languagetool.Language;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Scanner;
import java.util.regex.Pattern;

/**
 * Provides access to the relevant sentences of a plain text file
 * with one sentence per line.
 * @since 3.0
 */
public class PlainTextSentenceSource extends SentenceSource {

  private final List<String> sentences;
  private final Scanner scanner;

  // Each sentence is one article, but count anyway so it's coherent with what the Wikipedia code does:
  private int articleCount = 0;
  private String currentUrl = null;

  public PlainTextSentenceSource(InputStream textInput, Language language) {
    this(textInput, language, null);
  }
  
  /** @since 3.0 */
  public PlainTextSentenceSource(InputStream textInput, Language language, Pattern filter) {
    super(language, filter);
    scanner = new Scanner(textInput);
    sentences = new ArrayList<>();
  }

  @Override
  public boolean hasNext() {
    fillSentences();
    return sentences.size() > 0;
  }

  @Override
  public Sentence next() {
    fillSentences();
    if (sentences.isEmpty()) {
      throw new NoSuchElementException();
    }
    return new Sentence(sentences.remove(0), getSource(), "<plaintext>", null, ++articleCount);
  }

  @Override
  public String getSource() {
    return currentUrl;
  }

  private void fillSentences() {
    while (sentences.isEmpty() && scanner.hasNextLine()) {
      String line = scanner.nextLine();
      if (line.isEmpty()) {
        continue;
      }
      if (line.startsWith("# source:")) {
        currentUrl = line.substring("# source: ".length());
        continue;
      }
      if (acceptSentence(line)) {
        sentences.add(line);
      }
    }
  }

{'fraction_non_alphanumeric': 0.09439305267132339, 'fraction_numerical': 0.009061733056447046, 'mean_word_length': 3.594969644405898, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4760528', 'n_tokens_mistral': 1502, 'n_tokens_neox': 1436, 'n_words': 555}