<commit_before>///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2000 <NAME> and <NAME>
// 
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
//////////////////////////////////////////////////////////////////////////////

package opennlp.common.xml;

import java.io.*;
import org.jdom.*;
import org.jdom.input.*;
import org.jdom.output.*;

/**
 * A class which wraps an XmlDocument inside and ensures that it fits OpenNLP
 * specifications.
 *
 * @author      <NAME>
 * @version     $Revision: 1.2 $, $Date: 2001/11/09 12:17:56 $
 */

public abstract class NLPDocument {
    protected Document nlpdoc;
    abstract String getDTD();
    
    public NLPDocument(String text) {
	String doc = getDTD() + "<NLPDOC>\n" +
	    "<TEXT>\n" + text + "</TEXT>\n" + "</NLPDOC>\n";

	try {
            nlpdoc = new SAXBuilder(true).build(new StringReader(doc));
	} catch (Exception e) {
	    System.out.println("Invalid XML Document.");
	    System.out.println(e.toString());
	}
	
    }

    public Document getNLPDoc() { return nlpdoc; }
    
    public String toString() {
	StringWriter sw = new StringWriter();
         
	  try { 
	      new XMLOutputter("  ", true).output(nlpdoc, sw);
	  }
	  catch (Exception e) {
	      System.out.println("Unable to print document.");
	      System.out.println(e);
	  } 
	
	return sw.toString();
    }
    
    public static void main (String[] args) {
	NLPDocument doc = new PreProcessDocument("Here is a sentence");
	System.out.println(doc.toString());
    }
}<commit_msg>Added methods to create NLP document elements TOK and SENT.<commit_after>///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2000 <NAME> and <NAME>
// 
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
//////////////////////////////////////////////////////////////////////////////

package opennlp.common.xml;

import java.io.*;
import org.jdom.*;
import org.jdom.input.*;
import org.jdom.output.*;

/**
 * A class which wraps an XmlDocument inside and ensures that it fits OpenNLP
 * specifications.
 *
 * @author      <NAME>
 * @version     $Revision: 1.3 $, $Date: 2001/11/16 14:18:47 $
 */

public abstract class NLPDocument {
    protected Document nlpdoc;
    abstract String getDTD();
    
    public NLPDocument(String text) {
	String doc = getDTD() + "<NLPDOC>\n" +
	    "<TEXT>\n" + text + "</TEXT>\n" + "</NLPDOC>\n";

	try {
            nlpdoc = new SAXBuilder(true).build(new StringReader(doc));
	} catch (Exception e) {
	    System.out.println("Invalid XML Document.");
	    System.out.println(e.toString());
	}
	
    }

    public Document getNLPDoc() { return nlpdoc; }
    
    public String toString() {
	StringWriter sw = new StringWriter();
         
	  try { 
	      new XMLOutputter("  ", true).output(nlpdoc, sw);
	  }
	  catch (Exception e) {
	      System.out.println("Unable to print document.");
	      System.out.println(e);
	  } 
	
	return sw.toString();
    }

    
    /**
     * Create a SENT element for an NLP Document.
     * 
     * @param sent the string of text associated with this sentence.
     */
    public static Element createSENT(String sent) {
        int l = sent.length();
        Element sentEl = new Element("SENT");
        if (l>0) {
            char delim = sent.charAt(l-1);
            if(delim == '.' || delim == '?' || delim == '!') {
                sentEl.addContent(createTOK(sent.substring(0,l-1)));
                sentEl.addContent(createTOK(""+delim));
            }
	    else {
                sentEl.addContent(createTOK(sent));
	    }
        }
	return sentEl;
    }


    /**
     * Create a TOK element for an NLP Document.
     * 
     * @param tok the string of text associated with this token.
     */
    public static Element createTOK(String tok) {
        Element tokEl = new Element("TOK");
        Element lexEl = new Element("LEX");
	lexEl.setText(tok);
        tokEl.addContent(lexEl);
        return tokEl;
    }


    public static void main (String[] args) {
	NLPDocument doc = new PreProcessDocument("Here is a sentence");
	System.out.println(doc.toString());
    }
}
{'fraction_non_alphanumeric': 0.16981132075471697, 'fraction_numerical': 0.013788098693759071, 'mean_word_length': 2.1811886901327178, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6977302', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1510, 'n_words': 544}