<commit_before>import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.semgraph.SemanticGraph;
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations;
import edu.stanford.nlp.trees.TypedDependency;
import edu.stanford.nlp.util.CoreMap;

public class Extract {

    final static String ParserModelPath = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";

    public static void main(String[] args) {
        try {
            Extract.run(new FileReader("d:/review.txt"));
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
    }

    public static List<Pattern> run(Reader text) {

        LexicalizedParser parser = LexicalizedParser.loadModel(ParserModelPath);
        TreebankLanguagePack tlp = new PennTreebankLanguagePack();
        GrammaticalStructureFactory gsf = tlp.grammaticalStructureFactory();

        List<Pattern> patterns = new ArrayList<Pattern>();
        for (List<HasWord> sentence : new DocumentPreprocessor(text)) {
            patterns.addAll(ExtractSentencePatterns(parser, gsf, sentence));
        }

        return patterns;
    }

    private static HashSet<Pattern> ExtractSentencePatterns(LexicalizedParser parser, GrammaticalStructureFactory gsf, List<HasWord> sentence) {
        Tree parse = parser.apply(sentence);<commit_msg>replaced stanford parser with core nlp library<commit_after>import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.semgraph.SemanticGraph;
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations;
import edu.stanford.nlp.trees.TypedDependency;
import edu.stanford.nlp.util.CoreMap;

import java.util.*;

public class Extract {

    public List<Pattern> run(String text) {
        List<Pattern> patterns = new ArrayList<Pattern>();

        Properties props = new Properties();
        props.setProperty("annotators", "tokenize, ssplit, pos, parse");
        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
        Annotation annotation = pipeline.process(text);
        List<CoreMap> sentences = annotation.get(CoreAnnotations.SentencesAnnotation.class);
        for (CoreMap sentence : sentences) {
            patterns.addAll(ExtractSentencePatterns(sentence));
        }

        return patterns;
    }

    private static HashSet<Pattern> ExtractSentencePatterns(LexicalizedParser parser, GrammaticalStructureFactory gsf, List<HasWord> sentence) {
        Tree parse = parser.apply(sentence);
{'fraction_non_alphanumeric': 0.09639468690702087, 'fraction_numerical': 0.0, 'mean_word_length': 4.040152963671128, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4692263', 'n_tokens_mistral': 776, 'n_tokens_neox': 758, 'n_words': 145}