<commit_before>/*******************************************************************************
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *******************************************************************************/
package reciter.algorithm.evidence.targetauthor.department.strategy;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import reciter.algorithm.cluster.article.scorer.ReCiterArticleScorer;
import reciter.algorithm.evidence.targetauthor.AbstractTargetAuthorStrategy;
import reciter.engine.Feature;
import reciter.engine.analysis.evidence.OrganizationalUnitEvidence;
import reciter.model.article.ReCiterArticle;
import reciter.model.article.ReCiterAuthor;
import reciter.model.identity.AuthorName;
import reciter.model.identity.Identity;
import reciter.model.identity.OrganizationalUnit;

/**
 * This strategy matches on Organizational Unit recorded in Identity to affiliation string of target author
 * @author <NAME>(szd2013)
 * @see <a href ="https://github.com/wcmc-its/ReCiter/issues/229">Organizational Unit Strategy</a>
 *
 */
public class DepartmentStringMatchStrategy extends AbstractTargetAuthorStrategy {

	private final static Logger slf4jLogger = LoggerFactory.getLogger(DepartmentStringMatchStrategy.class);
	private final List<String> orgUnitSynonym = Arrays.asList(ReCiterArticleScorer.strategyParameters.getOrganizationalUnitSynonym().trim().split("\\s*,\\s*"));

	private String extractedDept;
	private long pmid;
	private int isGoldStandard;
	//	private Set<String> departments = new HashSet<String>();

	@Override
	public double executeStrategy(ReCiterArticle reCiterArticle, Identity identity) {

		pmid = reCiterArticle.getArticleId();
		isGoldStandard = reCiterArticle.getGoldStandard();

		double score = 0;
		if (reCiterArticle.getArticleCoAuthors() != null && reCiterArticle.getArticleCoAuthors().getAuthors() != null) {
			for (ReCiterAuthor author : reCiterArticle.getArticleCoAuthors().getAuthors()) {

				//				boolean isDepartmentMatch = departmentMatchStrict(author, targetAuthor);
				boolean isDepartmentMatch = departmentMatchStrictAndFillInAffiliationIfNotPresent(reCiterArticle.getArticleId(), reCiterArticle.getGoldStandard(),
						reCiterArticle.getArticleCoAuthors().getAuthors(), author, identity);

				boolean isFirstNameInitialMatch = 
						author.getAuthorName().getFirstInitial().equalsIgnoreCase(identity.getPrimaryName().getFirstInitial());

				boolean isFirstNameInitialMatchFromEmailFetched = false;
				if (identity.getAlternateNames() != null) {
					for (AuthorName authorName : identity.getAlternateNames()) {
						if (StringUtils.equalsIgnoreCase(authorName.getFirstInitial(), author.getAuthorName().getFirstInitial()) &&
								StringUtils.equalsIgnoreCase(authorName.getLastName(), author.getAuthorName().getLastName())) {
							isFirstNameInitialMatchFromEmailFetched = true;
							break;
						}
					}
				}

				if ((isDepartmentMatch && isFirstNameInitialMatch) || (isDepartmentMatch && isFirstNameInitialMatchFromEmailFetched)) {
					reCiterArticle.setClusterInfo(reCiterArticle.getClusterInfo() + 
							" [department and first name initial matches: " + extractedDept + 
							", first name initial: " + identity.getPrimaryName().getFirstInitial() + "]");
					slf4jLogger.info("Department and first name initial matches. "
							+ "PMID=[" + pmid + "] - Extracted Deptment From Article=[" + extractedDept + 
							"] Is Gold=[" + isGoldStandard + "]");
					score = 1;
					reCiterArticle.setMatchingDepartment(extractedDept);
					break;
				}
			}
		}
		reCiterArticle.setDepartmentStrategyScore(score);
		return score;
	}

	@Override
	public double executeStrategy(List<ReCiterArticle> reCiterArticles, Identity identity) {
		double sum = 0;
		Set<OrganizationalUnit> sanitizedIdentityInstitutions = new HashSet<OrganizationalUnit>();
		Map<String, List<String>> identityOrgUnitToSynonymMap = new HashMap<String, List<String>>();
		populateSanitizedIdentityInstitutions(identity, sanitizedIdentityInstitutions, identityOrgUnitToSynonymMap);
		for (ReCiterArticle reCiterArticle : reCiterArticles) {
			List<OrganizationalUnitEvidence> orgUnitEvidences = new ArrayList<OrganizationalUnitEvidence>(); 
			if (reCiterArticle.getArticleCoAuthors() != null && reCiterArticle.getArticleCoAuthors().getAuthors() != null) {
				for (ReCiterAuthor author : reCiterArticle.getArticleCoAuthors().getAuthors()) {
					if(author.isTargetAuthor() 
							&& 
							author.getAffiliation() != null) {
						
						String synonymOrgUnitLabel = null;
						String articleAffiliation = author.getAffiliation().replaceAll("&", "and").replaceAll("Tri-I", "Tri-Institutional").replaceAll("[-,]", "");
						if(identity.getOrganizationalUnits() != null 
								&& 
								identity.getOrganizationalUnits().size() > 0) {
							//This is for department
							for(OrganizationalUnit orgUnit: sanitizedIdentityInstitutions) {
								boolean isOrgUnitMatch = false;
								OrganizationalUnitEvidence orgUnitEvidence = new OrganizationalUnitEvidence();
								String identityDepartment = orgUnit.getOrganizationalUnitLabel().replaceAll("&", "and").replaceAll("Tri-I", "Tri-Institutional").replaceAll("[-,]", "");
								if(orgUnit.getOrganizationalUnitType().equals("department") || orgUnit.getOrganizationalUnitType().equals("division")) {
									if(orgUnit.getOrganizationalUnitLabel() != null 
											&& 
											(StringUtils.containsIgnoreCase(orgUnit.getOrganizationalUnitLabel(), "Center")
													||
													StringUtils.containsIgnoreCase(orgUnit.getOrganizationalUnitLabel(), "Program")
													||
													StringUtils.containsIgnoreCase(orgUnit.getOrganizationalUnitLabel(), "Institute")) 
											&& 
											orgUnit.getOrganizationalUnitLabel().length() > 14) {
										if(StringUtils.containsIgnoreCase(articleAffiliation, identityDepartment)) {
											//articleAffiliation: "Center for Integrative Medicine, Weill Cornell Medicine, New York, NY, USA."
											//identityDepartment: "Center for Integrative Medicine"
											//departmentMatchingScore: 2
											if(identityOrgUnitToSynonymMap.size() > 0 &&
													identityOrgUnitToSynonymMap.values().stream().anyMatch(synonymOrgUnit -> synonymOrgUnit.contains(identityDepartment))) {
													synonymOrgUnitLabel = identityOrgUnitToSynonymMap.entrySet().stream().
														filter(synonymOrgUnit -> synonymOrgUnit.getValue().contains(identityDepartment))
														.map(Map.Entry::getKey)
														.findFirst()
														.orElse(null);
													if(synonymOrgUnitLabel != null) {
														orgUnitEvidence.setIdentityOrganizationalUnit(synonymOrgUnitLabel);
													} 
											}
											if(synonymOrgUnitLabel == null) {
												orgUnitEvidence.setIdentityOrganizationalUnit(orgUnit.getOrganizationalUnitLabel());
											}
											orgUnitEvidence.setArticleAffiliation(author.getAffiliation());
											orgUnitEvidence.setOrganizationalUnitMatchingScore(ReCiterArticleScorer.strategyParameters.getOrganizationalUnitDepartmentMatchingScore());
											isOrgUnitMatch = true;
										}
									}  
									
									if(orgUnitEvidence.getOrganizationalUnitMatchingScore() == 0 
											&&
											StringUtils.containsIgnoreCase(articleAffiliation.replaceAll(constructRegexForStopWords(), ""), identityDepartment.replaceAll(constructRegexForStopWords(), ""))) {
										//This is for https://github.com/wcmc-its/ReCiter/issues/251 - Account for possibility that department name may be missing preposition in article.affiliation
										if(identityOrgUnitToSynonymMap.size() > 0 &&
												identityOrgUnitToSynonymMap.entrySet().stream().anyMatch(synonymOrgUnit -> synonymOrgUnit.getValue().contains(identityDepartment))) {
												synonymOrgUnitLabel = identityOrgUnitToSynonymMap.entrySet().stream().
													filter(synonymOrgUnit -> synonymOrgUnit.getValue().contains(identityDepartment))
													.map(Map.Entry::getKey)
													.findFirst()
													.orElse(null);
											if(synonymOrgUnitLabel != null) {
												orgUnitEvidence.setIdentityOrganizationalUnit(synonymOrgUnitLabel);
											} 
										}
										if(synonymOrgUnitLabel == null) {
											orgUnitEvidence.setIdentityOrganizationalUnit(orgUnit.getOrganizationalUnitLabel());
										}
										orgUnitEvidence.setArticleAffiliation(author.getAffiliation());
										orgUnitEvidence.setOrganizationalUnitMatchingScore(ReCiterArticleScorer.strategyParameters.getOrganizationalUnitDepartmentMatchingScore());
										isOrgUnitMatch = true;
									} else if(StringUtils.containsIgnoreCase(articleAffiliation, "Department of " + identityDepartment) 
											|| 
											StringUtils.containsIgnoreCase(articleAffiliation, "Division of " + identityDepartment)
											||
											StringUtils.containsIgnoreCase(articleAffiliation, "Dept of " + identityDepartment)
											||
											StringUtils.containsIgnoreCase(articleAffiliation, "Departments of " + identityDepartment)<commit_msg>department match bug fix <commit_after>/*******************************************************************************
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *******************************************************************************/
package reciter.algorithm.evidence.targetauthor.department.strategy;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import reciter.algorithm.cluster.article.scorer.ReCiterArticleScorer;
import reciter.algorithm.evidence.targetauthor.AbstractTargetAuthorStrategy;
import reciter.engine.Feature;
import reciter.engine.analysis.evidence.OrganizationalUnitEvidence;
import reciter.model.article.ReCiterArticle;
import reciter.model.article.ReCiterAuthor;
import reciter.model.identity.AuthorName;
import reciter.model.identity.Identity;
import reciter.model.identity.OrganizationalUnit;

/**
 * This strategy matches on Organizational Unit recorded in Identity to affiliation string of target author
 * @author <NAME>(szd2013)
 * @see <a href ="https://github.com/wcmc-its/ReCiter/issues/229">Organizational Unit Strategy</a>
 *
 */
public class DepartmentStringMatchStrategy extends AbstractTargetAuthorStrategy {

	private final static Logger slf4jLogger = LoggerFactory.getLogger(DepartmentStringMatchStrategy.class);
	private final List<String> orgUnitSynonym = Arrays.asList(ReCiterArticleScorer.strategyParameters.getOrganizationalUnitSynonym().trim().split("\\s*,\\s*"));

	private String extractedDept;
	private long pmid;
	private int isGoldStandard;
	//	private Set<String> departments = new HashSet<String>();

	@Override
	public double executeStrategy(ReCiterArticle reCiterArticle, Identity identity) {

		pmid = reCiterArticle.getArticleId();
		isGoldStandard = reCiterArticle.getGoldStandard();

		double score = 0;
		if (reCiterArticle.getArticleCoAuthors() != null && reCiterArticle.getArticleCoAuthors().getAuthors() != null) {
			for (ReCiterAuthor author : reCiterArticle.getArticleCoAuthors().getAuthors()) {

				//				boolean isDepartmentMatch = departmentMatchStrict(author, targetAuthor);
				boolean isDepartmentMatch = departmentMatchStrictAndFillInAffiliationIfNotPresent(reCiterArticle.getArticleId(), reCiterArticle.getGoldStandard(),
						reCiterArticle.getArticleCoAuthors().getAuthors(), author, identity);

				boolean isFirstNameInitialMatch = 
						author.getAuthorName().getFirstInitial().equalsIgnoreCase(identity.getPrimaryName().getFirstInitial());

				boolean isFirstNameInitialMatchFromEmailFetched = false;
				if (identity.getAlternateNames() != null) {
					for (AuthorName authorName : identity.getAlternateNames()) {
						if (StringUtils.equalsIgnoreCase(authorName.getFirstInitial(), author.getAuthorName().getFirstInitial()) &&
								StringUtils.equalsIgnoreCase(authorName.getLastName(), author.getAuthorName().getLastName())) {
							isFirstNameInitialMatchFromEmailFetched = true;
							break;
						}
					}
				}

				if ((isDepartmentMatch && isFirstNameInitialMatch) || (isDepartmentMatch && isFirstNameInitialMatchFromEmailFetched)) {
					reCiterArticle.setClusterInfo(reCiterArticle.getClusterInfo() + 
							" [department and first name initial matches: " + extractedDept + 
							", first name initial: " + identity.getPrimaryName().getFirstInitial() + "]");
					slf4jLogger.info("Department and first name initial matches. "
							+ "PMID=[" + pmid + "] - Extracted Deptment From Article=[" + extractedDept + 
							"] Is Gold=[" + isGoldStandard + "]");
					score = 1;
					reCiterArticle.setMatchingDepartment(extractedDept);
					break;
				}
			}
		}
		reCiterArticle.setDepartmentStrategyScore(score);
		return score;
	}

	@Override
	public double executeStrategy(List<ReCiterArticle> reCiterArticles, Identity identity) {
		double sum = 0;
		Set<OrganizationalUnit> sanitizedIdentityInstitutions = new HashSet<OrganizationalUnit>();
		Map<String, List<String>> identityOrgUnitToSynonymMap = new HashMap<String, List<String>>();
		populateSanitizedIdentityInstitutions(identity, sanitizedIdentityInstitutions, identityOrgUnitToSynonymMap);
		for (ReCiterArticle reCiterArticle : reCiterArticles) {
			List<OrganizationalUnitEvidence> orgUnitEvidences = new ArrayList<OrganizationalUnitEvidence>(); 
			if (reCiterArticle.getArticleCoAuthors() != null && reCiterArticle.getArticleCoAuthors().getAuthors() != null) {
				for (ReCiterAuthor author : reCiterArticle.getArticleCoAuthors().getAuthors()) {
					if(author.isTargetAuthor() 
							&& 
							author.getAffiliation() != null) {
						
						String synonymOrgUnitLabel = null;
						String articleAffiliation = author.getAffiliation().replaceAll("&", "and").replaceAll("Tri-I", "Tri-Institutional").replaceAll("[-,]", "");
						if(identity.getOrganizationalUnits() != null 
								&& 
								identity.getOrganizationalUnits().size() > 0) {
							//This is for department
							for(OrganizationalUnit orgUnit: sanitizedIdentityInstitutions) {
								boolean isOrgUnitMatch = false;
								OrganizationalUnitEvidence orgUnitEvidence = new OrganizationalUnitEvidence();
								String identityDepartment = orgUnit.getOrganizationalUnitLabel().replaceAll("&", "and").replaceAll("Tri-I", "Tri-Institutional").replaceAll("[-,]", "");
								if(orgUnit.getOrganizationalUnitType().equals("department") || orgUnit.getOrganizationalUnitType().equals("division")) {
									if(orgUnit.getOrganizationalUnitLabel() != null 
											&& 
											(StringUtils.containsIgnoreCase(orgUnit.getOrganizationalUnitLabel(), "Center")
													||
													StringUtils.containsIgnoreCase(orgUnit.getOrganizationalUnitLabel(), "Program")
													||
													StringUtils.containsIgnoreCase(orgUnit.getOrganizationalUnitLabel(), "Institute")) 
											&& 
											orgUnit.getOrganizationalUnitLabel().length() > 14) {
										if(StringUtils.containsIgnoreCase(articleAffiliation.replaceAll(constructRegexForStopWords(), ""), identityDepartment.replaceAll(constructRegexForStopWords(), ""))) {
											//articleAffiliation: "Center for Integrative Medicine, Weill Cornell Medicine, New York, NY, USA."
											//identityDepartment: "Center for Integrative Medicine"
											//departmentMatchingScore: 2
											if(identityOrgUnitToSynonymMap.size() > 0 &&
													identityOrgUnitToSynonymMap.values().stream().anyMatch(synonymOrgUnit -> synonymOrgUnit.contains(identityDepartment))) {
													synonymOrgUnitLabel = identityOrgUnitToSynonymMap.entrySet().stream().
														filter(synonymOrgUnit -> synonymOrgUnit.getValue().contains(identityDepartment))
														.map(Map.Entry::getKey)
														.findFirst()
														.orElse(null);
													if(synonymOrgUnitLabel != null) {
														orgUnitEvidence.setIdentityOrganizationalUnit(synonymOrgUnitLabel);
													} 
											}
											if(synonymOrgUnitLabel == null) {
												orgUnitEvidence.setIdentityOrganizationalUnit(orgUnit.getOrganizationalUnitLabel());
											}
											orgUnitEvidence.setArticleAffiliation(author.getAffiliation());
											orgUnitEvidence.setOrganizationalUnitMatchingScore(ReCiterArticleScorer.strategyParameters.getOrganizationalUnitDepartmentMatchingScore());
											isOrgUnitMatch = true;
										}
									}  
									
									/*if(orgUnitEvidence.getOrganizationalUnitMatchingScore() == 0 
											&&
											StringUtils.containsIgnoreCase(articleAffiliation.replaceAll(constructRegexForStopWords(), ""), identityDepartment.replaceAll(constructRegexForStopWords(), ""))) {
										//This is for https://github.com/wcmc-its/ReCiter/issues/251 - Account for possibility that department name may be missing preposition in article.affiliation
										if(identityOrgUnitToSynonymMap.size() > 0 &&
												identityOrgUnitToSynonymMap.entrySet().stream().anyMatch(synonymOrgUnit -> synonymOrgUnit.getValue().contains(identityDepartment))) {
												synonymOrgUnitLabel = identityOrgUnitToSynonymMap.entrySet().stream().
													filter(synonymOrgUnit -> synonymOrgUnit.getValue().contains(identityDepartment))
													.map(Map.Entry::getKey)
													.findFirst()
													.orElse(null);
											if(synonymOrgUnitLabel != null) {
												orgUnitEvidence.setIdentityOrganizationalUnit(synonymOrgUnitLabel);
											} 
										}
										if(synonymOrgUnitLabel == null) {
											orgUnitEvidence.setIdentityOrganizationalUnit(orgUnit.getOrganizationalUnitLabel());
										}
										orgUnitEvidence.setArticleAffiliation(author.getAffiliation());
										orgUnitEvidence.setOrganizationalUnitMatchingScore(ReCiterArticleScorer.strategyParameters.getOrganizationalUnitDepartmentMatchingScore());
										isOrgUnitMatch = true;
									}*/ else if(StringUtils.containsIgnoreCase(articleAffiliation, "Department of " + identityDepartment) 
											|| 
											StringUtils.containsIgnoreCase(articleAffiliation, "Division of " + identityDepartment)
											||
											StringUtils.containsIgnoreCase(articleAffiliation, "Dept of " + identityDepartment)
											||
											StringUtils.containsIgnoreCase(articleAffiliation, "Departments of " + identityDepartment)
{'fraction_non_alphanumeric': 0.115568570034163, 'fraction_numerical': 0.0028306490971205466, 'mean_word_length': 4.152376162936887, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 49, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3234533', 'n_tokens_mistral': 7423, 'n_tokens_neox': 5427, 'n_words': 993}