<commit_before>///////////////////////////////////////////////////////////////////////////////
//  Copyright (C) 2010 <NAME>, <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////
package opennlp.textgrounder.textstructs;

import java.io.*;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.List;

import org.jdom.*;
import org.jdom.input.SAXBuilder;

import opennlp.textgrounder.ners.*;
import opennlp.textgrounder.topostructs.Coordinate;
import opennlp.textgrounder.topostructs.Location;

/**
 * Class of static methods that read from a file containing text in some format,
 * possibly along with tags identifying whether they are toponyms (i.e.
 * locations) or other such information; generates a CorpusDocument object; and
 * populates it with tokens (mostly words, but can be multiword place-name
 * collocations). Each token is identified as to whether it is a toponym and
 * possibly also what type of token it is from a named-entity-recognition (NER)
 * perspective. If the text doesn't explicitly indicate which words are
 * toponyms, a named-entity recognizer (NER) is run to determine this info.
 * 
 * In addition, the CorpusDocument object has a mid-level structure made up of
 * Division objects that reflect the internal structure of the source file (e.g.
 * chapters, paragraphs, sentences). Furthermore, each CorpusDocument is itself
 * part of a Corpus (i.e. a collection of documents with common Lexicon and
 * Gazetteer objects). they are locations (toponyms), possibly with additional
 * properties. Tokens are added to a Document object.
 * 
 * Multi-word location tokens (e.g. New York) are joined by the code below into
 * a single token.
 * 
 * @author <NAME> partially based on earlier code by <NAME>
 */
public abstract class TextProcessor {
    /**
     * Process a raw-text file, running it through an NER to identify toponyms.
     * See comment at top of class.
     * 
     * This function works by splitting any incoming document into smaller
     * subdocuments based on parAsDocSize. These subdocuments are then passed to
     * the NER to do the actual work of generating Token objects and populating
     * the CorpusDocument with them.
     * 
     * @param locationOfFile
     *            Path to input. Must be a single file.
     * @param corpus
     * @param ner
     * @param parAsDocSize
     * @throws FileNotFoundException
     * @throws IOException
     */
    public static void processNER(String locationOfFile, Corpus corpus,
            NamedEntityRecognizer ner, int parAsDocSize) {

        try {
            CorpusDocument doc = new CorpusDocument(corpus, locationOfFile);
            BufferedReader textIn = new BufferedReader(new FileReader(
                    locationOfFile));
            System.out.println("Extracting toponym indices from "
                    + locationOfFile + " ...");

            String curLine = null;
            StringBuffer buf = new StringBuffer();
            int counter = 1;
            int currentDoc = 0;
            System.err.print("Processing document:" + locationOfFile + ",");
            while (true) {
                curLine = textIn.readLine();
                if (curLine == null || curLine.equals("")) {
                    break;
                }
                buf.append(curLine.replaceAll("[<>]", "")
                        .replaceAll("&", "and"));
                buf.append(" ");

                if (counter < parAsDocSize) {
                    counter++;
                } else {
                    ner.processText(doc, buf.toString());

                    buf = new StringBuffer();
                    currentDoc += 1;
                    counter = 1;
                    System.err.print(currentDoc + ",");
                }
            }

            /**
             * Add last lines if they have not been processed and added
             */
            if (counter > 1) {
                ner.processText(doc, buf.toString());
                System.err.print(currentDoc + ",");
            }
            System.err.println();

            corpus.add(doc);
        } catch (IOException ex) {
            throw new RuntimeException(ex);
        }
    }

    /**<commit_msg>Quick fix for TextProcessor parAsDocSize stuff.<commit_after>///////////////////////////////////////////////////////////////////////////////
//  Copyright (C) 2010 <NAME>, <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////
package opennlp.textgrounder.textstructs;

import java.io.*;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.List;

import org.jdom.*;
import org.jdom.input.SAXBuilder;

import opennlp.textgrounder.ners.*;
import opennlp.textgrounder.topostructs.Coordinate;
import opennlp.textgrounder.topostructs.Location;

/**
 * Class of static methods that read from a file containing text in some format,
 * possibly along with tags identifying whether they are toponyms (i.e.
 * locations) or other such information; generates a CorpusDocument object; and
 * populates it with tokens (mostly words, but can be multiword place-name
 * collocations). Each token is identified as to whether it is a toponym and
 * possibly also what type of token it is from a named-entity-recognition (NER)
 * perspective. If the text doesn't explicitly indicate which words are
 * toponyms, a named-entity recognizer (NER) is run to determine this info.
 * 
 * In addition, the CorpusDocument object has a mid-level structure made up of
 * Division objects that reflect the internal structure of the source file (e.g.
 * chapters, paragraphs, sentences). Furthermore, each CorpusDocument is itself
 * part of a Corpus (i.e. a collection of documents with common Lexicon and
 * Gazetteer objects). they are locations (toponyms), possibly with additional
 * properties. Tokens are added to a Document object.
 * 
 * Multi-word location tokens (e.g. New York) are joined by the code below into
 * a single token.
 * 
 * @author <NAME> partially based on earlier code by <NAME>
 */
public abstract class TextProcessor {
    /**
     * Process a raw-text file, running it through an NER to identify toponyms.
     * See comment at top of class.
     * 
     * This function works by splitting any incoming document into smaller
     * subdocuments based on parAsDocSize. These subdocuments are then passed to
     * the NER to do the actual work of generating Token objects and populating
     * the CorpusDocument with them.
     * 
     * @param locationOfFile
     *            Path to input. Must be a single file.
     * @param corpus
     * @param ner
     * @param parAsDocSize
     * @throws FileNotFoundException
     * @throws IOException
     */
    public static void processNER(String locationOfFile, Corpus corpus,
            NamedEntityRecognizer ner, int parAsDocSize) {

        try {
            CorpusDocument doc = new CorpusDocument(corpus, locationOfFile);
            BufferedReader textIn = new BufferedReader(new FileReader(
                    locationOfFile));
            System.out.println("Extracting toponym indices from "
                    + locationOfFile + " ...");

            String curLine = null;
            StringBuffer buf = new StringBuffer();
            int counter = 1;
            int currentDoc = 0;
            System.err.print("Processing document:" + locationOfFile + ",");
            while (true) {
                curLine = textIn.readLine();
                if (curLine == null || curLine.equals("")) {
                    break;
                }
                buf.append(curLine.replaceAll("[<>]", "")
                        .replaceAll("&", "and"));
                buf.append(" ");

                if (counter < parAsDocSize) {
                    counter++;
                } else {
                    ner.processText(doc, buf.toString());
                    corpus.add(doc);
                    doc = new CorpusDocument(corpus, locationOfFile);

                    buf = new StringBuffer();
                    currentDoc += 1;
                    counter = 1;
                    System.err.print(currentDoc + ",");
                }
            }

            /**
             * Add last lines if they have not been processed and added
             */
            if (counter > 1) {
                ner.processText(doc, buf.toString());
                System.err.print(currentDoc + ",");
            }
            System.err.println();

            corpus.add(doc);
        } catch (IOException ex) {
            throw new RuntimeException(ex);
        }
    }

    /**
{'fraction_non_alphanumeric': 0.11549781792347508, 'fraction_numerical': 0.0026387902161778137, 'mean_word_length': 1.9896844660194175, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7056113', 'n_tokens_mistral': 2477, 'n_tokens_neox': 2366, 'n_words': 938}