<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package de.hsbo.fbg.sm4c.classify.filter;

import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSTagger;
import opennlp.tools.postag.POSTaggerME;
import org.apache.logging.log4j.LogManager;

/**
 * Class for identifying the part of speech of a text
 *
 * @author <NAME>
 */
public class PartOfSpeechTagger {

    private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(PartOfSpeechTagger.class);

    public static final String DEFAULT_POS_MODEL = "de-pos-maxent.bin";

    private POSModel posModel;
    private POSTagger posTagger;

    /**
     * Trains the part of speech tagger with a given POS Model.
     *
     * @param modelPath path starting from the classpath to the model
     */
    public void trainPOSModel(String modelPath) {
        try (InputStream in = this.getClass().getClassLoader().getResourceAsStream(modelPath)) {
            posModel = new POSModel(in);
            posTagger = new POSTaggerME(posModel);
        } catch (IOException ex) {
            LOGGER.error("Could not load tokenizer model", ex);
        }
    }

    /**
     * Tags the part of speech for given tokens.
     *
     * @param tokens Tokens to tag
     * @return List of tags for the tokens
     */
    public List<String> tagTokens(List<String> tokens) {
        return Arrays.asList(posTagger.tag(tokens.toArray(new String[0])));
    }

}
<commit_msg>improve POS tagging<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package de.hsbo.fbg.sm4c.classify.filter;

import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.util.CoreMap;
import edu.stanford.nlp.util.PropertiesUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

/**
 *
 * @author <NAME>
 */
public class PartOfSpeechTagger {

    private StanfordCoreNLP pipeline;

    public PartOfSpeechTagger() {
        Properties props = PropertiesUtils.asProperties(
                "annotators", "tokenize, ssplit, pos",
                "ssplit.isOneSentence", "true",
                "pos.model", "edu/stanford/nlp/models/pos-tagger/german/german-hgc.tagger",
                "ner.model", "edu/stanford/nlp/models/ner/german.conll.hgc_175m_600.crf.ser.gz",
                "tokenize.language", "de",
                "ner.useSUTime", "false");
        this.pipeline = new StanfordCoreNLP(props);
    }

    public List<String> tagPOS(String text) {
        Annotation document = new Annotation(text);
        pipeline.annotate(document);
        List<CoreMap> sentences = document.get(CoreAnnotations.SentencesAnnotation.class);
        List<String> result = new ArrayList();

        sentences.forEach(s -> {
            s.get(CoreAnnotations.TokensAnnotation.class).forEach(t -> {
                String value = t.get(CoreAnnotations.PartOfSpeechAnnotation.class);
                result.add(value);
            });

        });

//        sentences.forEach(s -> {
//            s.get(CoreAnnotations.MentionsAnnotation.class).forEach(em -> {
//                String value = em.get(CoreAnnotations.TextAnnotation.class);
//                String tag = em.get(CoreAnnotations.NamedEntityTagAnnotation.class);
//                result.add(new NamedEntity(value, tag));
//            });
//        });
        return result;
    }

}

{'fraction_non_alphanumeric': 0.11029796131730267, 'fraction_numerical': 0.002875065342394145, 'mean_word_length': 2.789108910891089, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3232425', 'n_tokens_mistral': 1156, 'n_tokens_neox': 1132, 'n_words': 261}