<commit_before>package intro;

import java.io.IOException;
import java.io.PrintWriter;
import java.util.Properties;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;

class SimpleExample {
    private static final StanfordCoreNLP pipeline;

    static {
        final Properties props = new Properties();

        // tokenize: split the text in tokens (words and punctuation)
        // ssplit: group the above tokens in sentences according to punctuation
        // pos: tags words as nouns, verbs etc
        props.put("annotators", "tokenize ssplit pos");

        // initialize the processing engine for the above annotators
        pipeline = new StanfordCoreNLP(props);
    }

    public static void main(String[] args) throws IOException {
        // an appropriate sample text for starting our journey
        final String text = "Space: the final frontier. " +
            "These are the voyages of the starship Enterprise. " +
            "Its continuing mission: to explore strange new worlds, " +
            "to seek out new life and new civilizations, " +
            "to boldly go where no one has gone before.";

        // the Annotation instance, containing both input and output of the processing
        final Annotation document = new Annotation(text);

        // process the text and store the result in Annotation instance
        pipeline.annotate(document);

        // output the annotation at stdout as a JSON
        pipeline.jsonPrint(document, new PrintWriter(System.out));
    }
}<commit_msg>improved comments<commit_after>package intro;

import java.io.IOException;
import java.io.PrintWriter;
import java.util.Properties;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;

class SimpleExample {
    private static final StanfordCoreNLP pipeline;

    static {
        final Properties props = new Properties();

        // tokenize: split the text in tokens (words and punctuation)
        // ssplit: group the above tokens in sentences according to punctuation
        // pos: tags words as nouns, verbs etc
        props.put("annotators", "tokenize ssplit pos");

        // initialize the processing engine for the above annotators
        pipeline = new StanfordCoreNLP(props);
    }

    public static void main(String[] args) throws IOException {
        // appropriate sample text for starting our journey
        final String text = "Space: the final frontier. " +
            "These are the voyages of the starship Enterprise. " +
            "Its continuing mission: to explore strange new worlds, " +
            "to seek out new life and new civilizations, " +
            "to boldly go where no one has gone before.";
        // cue pompous music after the above

        // the Annotation instance, containing both input and output of the processing
        final Annotation document = new Annotation(text);

        // process the text and store the result in Annotation instance
        pipeline.annotate(document);

        // output the annotation at stdout as a JSON
        pipeline.jsonPrint(document, new PrintWriter(System.out));
    }
}
{'fraction_non_alphanumeric': 0.07160804020100503, 'fraction_numerical': 0.0, 'mean_word_length': 2.7295081967213113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4930794', 'n_tokens_mistral': 795, 'n_tokens_neox': 773, 'n_words': 324}