<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package de.hsbo.fbg.sm4c.classify.geotag;

import de.hsbo.fbg.common.config.Configuration;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.util.CoreMap;
import edu.stanford.nlp.util.PropertiesUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.geonames.InvalidParameterException;
import org.geonames.Toponym;
import org.geonames.ToponymSearchCriteria;
import org.geonames.ToponymSearchResult;
import org.geonames.WebService;

/**
 *
 * @author <NAME>
 */
public class LocationRecognizer {

    private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(LocationRecognizer.class);

    private StanfordCoreNLP pipeline;

    public LocationRecognizer() {
        Properties props = PropertiesUtils.asProperties(
                "annotators", "tokenize, ssplit, pos, lemma, ner, entitymentions, parse",
                "ssplit.isOneSentence", "true",
                "parse.model", "edu/stanford/nlp/models/lexparser/germanFactored.ser.gz",
                "pos.model", "edu/stanford/nlp/models/pos-tagger/german/german-hgc.tagger",
                "ner.model", "edu/stanford/nlp/models/ner/german.conll.hgc_175m_600.crf.ser.gz",
                "tokenize.language", "de",
                "ner.useSUTime", "false");
        this.pipeline = new StanfordCoreNLP(props);
        WebService.setUserName(Configuration.getConfig().getPropertyValue("geonames_user"));
    }

    public List<String> recognizeLocations(String text) {
        List<String> result = new ArrayList();
        
        List<NamedEntity> entities = tagNamedEntities(text);
        entities.forEach(e -> {
            //if named entity annotation has tagged value as location,
            //add the value to the candidate list of locations
            if (e.getTag().equals("LOC")){
                result.add(e.getValue());
            }
            //otherwise check if the value is indexed in geonames.org
            //and add it to candidates if true
            else{
                if(isLocation(e.getValue())){
                    result.add(e.getValue());
                }
            }
        });

        return result;
    }

    private boolean isLocation(String entity) {
        boolean result = false;
        try {
            ToponymSearchCriteria searchCriteria = new ToponymSearchCriteria();
            searchCriteria.setNameEquals(entity);
            searchCriteria.setCountryCode("DE");
            searchCriteria.setMaxRows(1);
            ToponymSearchResult searchResult = WebService.search(searchCriteria);
            List<Toponym> topos = searchResult.getToponyms();
            result = !topos.isEmpty();
        } catch (InvalidParameterException ex) {
            LOGGER.error("Country code is not supported", ex);
        } catch (Exception ex) {
            LOGGER.error("Search for geoname failed", ex);
        }
        return result;
    }

    private List<NamedEntity> tagNamedEntities(String text) {
        Annotation document = new Annotation(text);
        pipeline.annotate(document);
        List<CoreMap> sentences = document.get(CoreAnnotations.SentencesAnnotation.class);
        List<NamedEntity> result = new ArrayList();

        sentences.forEach(s -> {
            s.get(CoreAnnotations.MentionsAnnotation.class).forEach(em -> {
                String value = em.get(CoreAnnotations.TextAnnotation.class);
                String tag = em.get(CoreAnnotations.NamedEntityTagAnnotation.class);
                result.add(new NamedEntity(value, tag));
            });
        });
        return result;
    }

}
<commit_msg>adjust NER processing due to performance problems<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package de.hsbo.fbg.sm4c.classify.geotag;

import de.hsbo.fbg.common.config.Configuration;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.util.CoreMap;
import edu.stanford.nlp.util.PropertiesUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.logging.log4j.LogManager;
import org.geonames.InvalidParameterException;
import org.geonames.Toponym;
import org.geonames.ToponymSearchCriteria;
import org.geonames.ToponymSearchResult;
import org.geonames.WebService;

/**
 *
 * @author <NAME>
 */
public class LocationRecognizer {

    private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(LocationRecognizer.class);

    private StanfordCoreNLP pipeline;

    public LocationRecognizer() {
        Properties props = PropertiesUtils.asProperties(
                "annotators", "tokenize, ssplit, pos, lemma, ner, entitymentions",
                "ssplit.isOneSentence", "true",
                "pos.model", "edu/stanford/nlp/models/pos-tagger/german/german-hgc.tagger",
                "ner.model", "edu/stanford/nlp/models/ner/german.conll.hgc_175m_600.crf.ser.gz",
                "tokenize.language", "de",
                "ner.useSUTime", "false");
        this.pipeline = new StanfordCoreNLP(props);
        WebService.setUserName(Configuration.getConfig().getPropertyValue("geonames_user"));
    }

    public List<String> recognizeLocations(String text) {
        List<String> result = new ArrayList();
        
        List<NamedEntity> entities = tagNamedEntities(text);
        entities.forEach(e -> {
            //if named entity annotation has tagged value as location,
            //add the value to the candidate list of locations
            if (e.getTag().equals("LOC")){
                result.add(e.getValue());
            }
            //otherwise check if the value is indexed in geonames.org
            //and add it to candidates if true
            else if (isLocation(e.getValue())) {
                result.add(e.getValue());
            }
        });

        return result;
    }

    private boolean isLocation(String entity) {
        boolean result = false;
        try {
            ToponymSearchCriteria searchCriteria = new ToponymSearchCriteria();
            searchCriteria.setNameEquals(entity);
            searchCriteria.setCountryCode("DE");
            searchCriteria.setMaxRows(1);
            ToponymSearchResult searchResult = WebService.search(searchCriteria);
            List<Toponym> topos = searchResult.getToponyms();
            result = !topos.isEmpty();
        } catch (InvalidParameterException ex) {
            LOGGER.error("Country code is not supported", ex);
        } catch (Exception ex) {
            LOGGER.error("Search for geoname failed", ex);
        }
        return result;
    }

    private List<NamedEntity> tagNamedEntities(String text) {
        Annotation document = new Annotation(text);
        pipeline.annotate(document);
        List<CoreMap> sentences = document.get(CoreAnnotations.SentencesAnnotation.class);
        List<NamedEntity> result = new ArrayList();

        sentences.forEach(s -> {
            s.get(CoreAnnotations.MentionsAnnotation.class).forEach(em -> {
                String value = em.get(CoreAnnotations.TextAnnotation.class);
                String tag = em.get(CoreAnnotations.NamedEntityTagAnnotation.class);
                result.add(new NamedEntity(value, tag));
            });
        });
        return result;
    }

}

{'fraction_non_alphanumeric': 0.10529666412019353, 'fraction_numerical': 0.002546473134708429, 'mean_word_length': 2.6181483187471213, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6747623', 'n_tokens_mistral': 2191, 'n_tokens_neox': 2145, 'n_words': 458}