<commit_before>package de.berlin.hu.uima.ae;

import de.berlin.hu.util.Constants;
import org.apache.uima.analysis_component.JCasAnnotator_ImplBase;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.jcas.JCas;
import org.u_compare.shared.semantic.NamedEntity;
import org.u_compare.shared.semantic.chemical.Chemical;
import org.u_compare.shared.syntactic.Sentence;
import org.uimafit.util.JCasUtil;

import java.util.*;
import java.util.regex.Pattern;

/**
 * Merges annotations from the CRF and the dictionary, favoring entities extracted by the CRF over the dictionary.
 */
public class AnnotationMergerAE extends JCasAnnotator_ImplBase {

	/* (non-Javadoc)
	 * @see org.apache.uima.analysis_component.JCasAnnotator_ImplBase#process(org.apache.uima.jcas.JCas)
	 */
	@Override
	public void process(JCas aJCas) throws AnalysisEngineProcessException {
		Iterator<Sentence> sentenceIterator = JCasUtil.iterator(aJCas, Sentence.class);
		
		while (sentenceIterator.hasNext()) {
			Sentence sentence = (Sentence) sentenceIterator.next();
			
			Iterator<NamedEntity> entityIterator = JCasUtil.iterator(sentence, NamedEntity.class, true, true);
			
			List<NamedEntity> entities = new ArrayList<NamedEntity>();
			List<String> abbreviations = new ArrayList<String>();
			Map<String, List<NamedEntity>> chemicalsMap = new HashMap<String, List<NamedEntity>>();
			while (entityIterator.hasNext()) {
				NamedEntity namedEntity = (NamedEntity) entityIterator.next();

				if (!Constants.GOLDSTANDARD.equals(namedEntity.getSource())) {
					entities.add(namedEntity);
					
					String chemName = namedEntity.getCoveredText();
					if (!chemicalsMap.containsKey(chemName)) {
						chemicalsMap.put(chemName, new ArrayList<NamedEntity>());
					}
					chemicalsMap.get(chemName).add(namedEntity);
				}
			}
			
			Comparator<NamedEntity> comp = new Comparator<NamedEntity>() {
				public int compare(NamedEntity m1, NamedEntity m2) {
					return m1.getBegin() - m2.getBegin(); 
				}
			};
				
			Collections.sort(entities, comp);
			
			List<NamedEntity> chemicals = new ArrayList<NamedEntity>();
			boolean filtered = false;
			
			//FIXME: use drug if it is identical to CRF match to obtain CAS Registry ID
			NamedEntity lastEntity = null;
			for (NamedEntity entity : entities) {
				boolean isChemAbbreviation = abbreviations.contains(entity.getCoveredText());
				if (Constants.ABBREV.equals(entity.getSource()) && !isChemAbbreviation) {
					String name = entity.getId();
					entity.setId(null);
					
					if (chemicalsMap.containsKey(name)) {
						List<NamedEntity> chems = chemicalsMap.get(name);
						
						for (NamedEntity c : chems) {
							if (!Constants.ABBREV.equals(c.getSource())) {
								isChemAbbreviation = true;
								break;
							}
						}
					}
					
					if (isChemAbbreviation) {
						abbreviations.add(entity.getCoveredText());
					} else {
						entity.removeFromIndexes(aJCas);
						filtered = true;
					}
				}
				
				if (!filtered && lastEntity != null && crosses(lastEntity, entity)) {					
					if (Constants.ABBREV.equals(lastEntity.getSource())) {
						if (isReplaceByAbbreviation(lastEntity, entity) && (!Constants.ABBREV.equals(entity.getSource()) || isChemAbbreviation)) {
							//System.out.printf("replacing %s annotation %s at [%d-%d], because it was identified as an abbreviation for %s%n", entity.getSource(), entity.getCoveredText(), entity.getBegin(), entity.getEnd(), lastEntity.getId());
							entity.removeFromIndexes(aJCas);
							filtered = true;
						}
					} else if (Constants.ABBREV.equals(entity.getSource())) {
						boolean isRemove = false;
						
						if (isChemAbbreviation && isReplaceByAbbreviation(entity, lastEntity)) {
							//System.out.printf("replacing %s annotation %s at [%d-%d], because it was identified as an abbreviation for %s%n", lastEntity.getSource(), lastEntity.getCoveredText(), lastEntity.getBegin(), lastEntity.getEnd(), entity.getId());
							isRemove = true;
						} else if (!isChemAbbreviation) {
							//System.out.printf("removing %s annotation %s at [%d-%d], because it does not appear to be a chemical abbreviation (%s = %s)%n", lastEntity.getSource(), lastEntity.getCoveredText(), lastEntity.getBegin(), lastEntity.getEnd(), entity.getCoveredText(), entity.getId());
							isRemove = true;
						}
						
						if (isRemove) {
							lastEntity.removeFromIndexes(aJCas);
							chemicals.remove(lastEntity);
							filtered = true;
						}
					} else if (Constants.DICTIONARY.equals(lastEntity.getSource()) 
							&& !Constants.DICTIONARY.equals(entity.getSource())) {
						lastEntity.removeFromIndexes(aJCas);
						chemicals.remove(lastEntity);
					} else if (!Constants.DICTIONARY.equals(lastEntity.getSource()) 
							&& Constants.DICTIONARY.equals(entity.getSource())) {
						entity.removeFromIndexes(aJCas);
						filtered = true;
					} else if (Constants.SUM_TAGGER.equals(lastEntity.getSource()) 
							&& !Constants.SUM_TAGGER.equals(entity.getSource())) {<commit_msg>significantly improved annotation merging of abbreviations<commit_after>package de.berlin.hu.uima.ae;

import de.berlin.hu.types.PubmedDocument;
import de.berlin.hu.util.Constants;
import org.apache.uima.analysis_component.JCasAnnotator_ImplBase;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.jcas.JCas;
import org.u_compare.shared.semantic.NamedEntity;
import org.u_compare.shared.semantic.chemical.Chemical;
import org.u_compare.shared.syntactic.Sentence;
import org.uimafit.util.JCasUtil;

import java.util.*;
import java.util.regex.Pattern;

/**
 * Merges annotations from the CRF and the dictionary, favoring entities extracted by the CRF over the dictionary.
 */
public class AnnotationMergerAE extends JCasAnnotator_ImplBase {

	/* (non-Javadoc)
	 * @see org.apache.uima.analysis_component.JCasAnnotator_ImplBase#process(org.apache.uima.jcas.JCas)
	 */
	@Override
	public void process(JCas aJCas) throws AnalysisEngineProcessException {
		List<PubmedDocument> documents = new ArrayList<PubmedDocument>();
		for (PubmedDocument doc : JCasUtil.iterate(aJCas, PubmedDocument.class)) {
			documents.add(doc);
		}
		
		if (documents.isEmpty()) {
			documents.add(null);
		}
		
		for (PubmedDocument document : documents) {
			//System.out.println("\nMerging annotations of document " + (document != null ? document.getPmid() : ""));
				
			Iterator<NamedEntity> entityIterator = document != null ? JCasUtil.iterator(document, NamedEntity.class, true, true) :  JCasUtil.iterator(aJCas, NamedEntity.class);
			
			List<NamedEntity> entities = new ArrayList<NamedEntity>();
			List<String> abbreviations = new ArrayList<String>();
			Map<String, List<NamedEntity>> chemicalsMap = new HashMap<String, List<NamedEntity>>();
			while (entityIterator.hasNext()) {
				NamedEntity namedEntity = (NamedEntity) entityIterator.next();

				if (!Constants.GOLDSTANDARD.equals(namedEntity.getSource())) {
					entities.add(namedEntity);
					
					String chemName = namedEntity.getCoveredText().trim().toLowerCase();
					if (!chemicalsMap.containsKey(chemName)) {
						chemicalsMap.put(chemName, new ArrayList<NamedEntity>());
					}
					chemicalsMap.get(chemName).add(namedEntity);
				}
			}
			
			Comparator<NamedEntity> comp = new Comparator<NamedEntity>() {
				public int compare(NamedEntity m1, NamedEntity m2) {
					return m1.getBegin() - m2.getBegin(); 
				}
			};
				
			Collections.sort(entities, comp);
			
			List<NamedEntity> chemicals = new ArrayList<NamedEntity>();
			boolean filtered = false;
			
			//FIXME: use drug if it is identical to CRF match to obtain CAS Registry ID
			NamedEntity lastEntity = null;
			List<String> nonChemicalAbbreviations = new ArrayList<String>();
			Iterator<Sentence> sentenceIterator = document != null ? JCasUtil.iterator(document, Sentence.class, true, true) :  JCasUtil.iterator(aJCas, Sentence.class);
			Sentence sentence = sentenceIterator.hasNext() ? sentenceIterator.next() : null;
			for (NamedEntity entity : entities) {
				if (sentence != null) {
					while (sentence.getEnd() < entity.getBegin() && sentenceIterator.hasNext()) {
						sentence = sentenceIterator.next();
						lastEntity = null;
					}
				}
				
				if (nonChemicalAbbreviations.contains(entity.getCoveredText().trim().toLowerCase())) {
					entity.removeFromIndexes(aJCas);
					filtered = true;
					//System.out.println("Filtered subsequent mention of non-Chemical abbreviation: " + entity.getCoveredText() + " -> " + entity.getId());
				}
				
				boolean isChemAbbreviation = abbreviations.contains(entity.getCoveredText().trim().toLowerCase());
				if (!isChemAbbreviation && !filtered && Constants.ABBREV.equals(entity.getSource())) {
					String name = entity.getId().trim().toLowerCase();
					entity.setId(null);
					
					if (chemicalsMap.containsKey(name)) {
						List<NamedEntity> chems = chemicalsMap.get(name);
						
						for (NamedEntity c : chems) {
							if (!Constants.ABBREV.equals(c.getSource())) {
								isChemAbbreviation = true;
								break;
							}
						}
					}
					
					if (isChemAbbreviation) {
						abbreviations.add(entity.getCoveredText().trim().toLowerCase());
					} else {
						entity.removeFromIndexes(aJCas);
						filtered = true;
						nonChemicalAbbreviations.add(entity.getCoveredText().trim().toLowerCase());
						
						List<NamedEntity> filteredEntities = new ArrayList<NamedEntity>();
						for (NamedEntity e : chemicals) {
							if (entity.getCoveredText().trim().equalsIgnoreCase(e.getCoveredText().trim())) {
								filteredEntities.add(e);
								e.removeFromIndexes();
								//System.out.println("Removing previous occurence of " + entity.getCoveredText());
							}
						}
						
						chemicals.removeAll(filteredEntities);
					}
				}
				
				if (!filtered && lastEntity != null && crosses(lastEntity, entity)) {					
					if (Constants.ABBREV.equals(lastEntity.getSource())) {
						if (isReplaceByAbbreviation(lastEntity, entity) && (!Constants.ABBREV.equals(entity.getSource()) || isChemAbbreviation)) {
							//System.out.printf("replacing %s annotation %s at [%d-%d], because it was identified as an abbreviation for %s%n", entity.getSource(), entity.getCoveredText(), entity.getBegin(), entity.getEnd(), lastEntity.getId());
							entity.removeFromIndexes(aJCas);
							filtered = true;
						}
					} else if (Constants.ABBREV.equals(entity.getSource())) {
						boolean isRemove = false;
						
						if (isChemAbbreviation && isReplaceByAbbreviation(entity, lastEntity)) {
							//System.out.printf("replacing %s annotation %s at [%d-%d], because it was identified as an abbreviation for %s%n", lastEntity.getSource(), lastEntity.getCoveredText(), lastEntity.getBegin(), lastEntity.getEnd(), entity.getId());
							isRemove = true;
						} else if (!isChemAbbreviation) {
							//System.out.printf("removing %s annotation %s at [%d-%d], because it does not appear to be a chemical abbreviation (%s = %s)%n", lastEntity.getSource(), lastEntity.getCoveredText(), lastEntity.getBegin(), lastEntity.getEnd(), entity.getCoveredText(), entity.getId());
							isRemove = true;
							filtered = true;
						}
						
						if (isRemove) {
							lastEntity.removeFromIndexes(aJCas);
							chemicals.remove(lastEntity);
						}
					} else if (Constants.DICTIONARY.equals(lastEntity.getSource()) 
							&& !Constants.DICTIONARY.equals(entity.getSource())) {
						lastEntity.removeFromIndexes(aJCas);
						chemicals.remove(lastEntity);
					} else if (!Constants.DICTIONARY.equals(lastEntity.getSource()) 
							&& Constants.DICTIONARY.equals(entity.getSource())) {
						entity.removeFromIndexes(aJCas);
						filtered = true;
					} else if (Constants.SUM_TAGGER.equals(lastEntity.getSource()) 
							&& !Constants.SUM_TAGGER.equals(entity.getSource())) {
{'fraction_non_alphanumeric': 0.12974203338391502, 'fraction_numerical': 0.0006744225257123588, 'mean_word_length': 3.814529220779221, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2493879', 'n_tokens_mistral': 4620, 'n_tokens_neox': 3515, 'n_words': 682}