<commit_before>package test.examples.pubmed;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import main.database.dao.ArticleDao;
import main.reciter.algorithm.cluster.ReCiterClusterer;
import main.reciter.lucene.DocumentIndexReader;
import main.reciter.lucene.DocumentIndexWriter;
import main.reciter.lucene.DocumentTranslator;
import main.reciter.model.article.ReCiterArticle;
import main.reciter.model.article.ReCiterArticleCoAuthors;
import main.reciter.model.article.ReCiterArticleKeywords;
import main.reciter.model.author.AuthorAffiliation;
import main.reciter.model.author.AuthorName;
import main.reciter.model.author.ReCiterAuthor;
import main.reciter.model.author.TargetAuthor;
import main.reciter.utils.Analysis;
import main.reciter.utils.AnalysisCSVWriter;
import main.reciter.utils.AnalysisObject;
import main.reciter.utils.ReCiterConfigProperty;
import main.xml.pubmed.PubmedXmlFetcher;
import main.xml.pubmed.model.MedlineCitationArticleAuthor;
import main.xml.pubmed.model.PubmedArticle;
import main.xml.scopus.ScopusXmlFetcher;
import main.xml.scopus.model.ScopusEntry;
import main.xml.translator.ArticleTranslator;

import org.apache.lucene.document.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ReCiterExample {

	private final static Logger slf4jLogger = LoggerFactory.getLogger(ReCiterExample.class);

	public static void main(String[] args) throws IOException {
		// Read Property file from config.properties.
		ReCiterConfigProperty reCiterConfigProperty = new ReCiterConfigProperty();
		reCiterConfigProperty.loadProperty("config.properties");

		// Keep track of exeuction time of ReCiter.
		long startTime = System.currentTimeMillis();

		// Run ReCiter.
		ReCiterExample reCiterExample = new ReCiterExample();
		reCiterExample.runExample(reCiterConfigProperty);

		long stopTime = System.currentTimeMillis();
		long elapsedTime = stopTime - startTime;
		slf4jLogger.info("Total execution time: " + elapsedTime + " ms.");
	}

	/**
	 * Setup the data to run the ReCiter algorithm.
	 * @param lastName
	 * @param firstInitial
	 * @param cwid
	 */
	public void runExample(ReCiterConfigProperty reCiterConfigProperty) {

		String lastName = reCiterConfigProperty.getLastName();
		String middleName = reCiterConfigProperty.getMiddleName();
		String firstName = reCiterConfigProperty.getFirstName();
		String affiliation = reCiterConfigProperty.getAuthorAffiliation();
		String firstInitial = firstName.substring(0, 1);
		String cwid = reCiterConfigProperty.getCwid();

		String authorKeywords = reCiterConfigProperty.getAuthorKeywords();
		String coAuthors = reCiterConfigProperty.getCoAuthors();
		double similarityThreshold = reCiterConfigProperty.getSimilarityThreshold();

		String department = reCiterConfigProperty.getAuthorDepartment();
		
		DocumentIndexWriter.setUseStopWords(reCiterConfigProperty.isUseStemming());
		
		// Define Singleton target author.
		TargetAuthor.init(new AuthorName(firstName, middleName, lastName), new AuthorAffiliation(affiliation));
		ReCiterArticle targetAuthorArticle = new ReCiterArticle(-1);
		targetAuthorArticle.setArticleCoAuthors(new ReCiterArticleCoAuthors());
		targetAuthorArticle.getArticleCoAuthors().addCoAuthor(new ReCiterAuthor(new AuthorName(firstName, middleName, lastName), new AuthorAffiliation(affiliation + " " + department)));		

		targetAuthorArticle.setArticleKeywords(new ReCiterArticleKeywords());
		for (String keyword : authorKeywords.split(",")) {
			targetAuthorArticle.getArticleKeywords().addKeyword(keyword);
		}

		for (String author : coAuthors.split(",")) {
			String[] authorArray = author.split(" ");

			if (authorArray.length == 2) {
				String coAuthorFirstName = authorArray[0];
				String coAuthorLastName = authorArray[1];
				targetAuthorArticle.getArticleCoAuthors().addCoAuthor(new ReCiterAuthor(new AuthorName(coAuthorFirstName, "", coAuthorLastName), new AuthorAffiliation("")));
			} else if (authorArray.length == 3) {
				String coAuthorFirstName = authorArray[0];
				String coAuthorMiddleName = authorArray[1];
				String coAuthorLastName = authorArray[2];
				targetAuthorArticle.getArticleCoAuthors().addCoAuthor(new ReCiterAuthor(new AuthorName(coAuthorFirstName, coAuthorMiddleName, coAuthorLastName), new AuthorAffiliation("")));
			}
		}
		
		
		// Try reading from Lucene Index:
		DocumentIndexReader documentIndexReader = new DocumentIndexReader();

		// Lucene Index doesn't contain this cwid's files.
		if (!documentIndexReader.isIndexed(cwid)) {

			slf4jLogger.debug("index...");

			// Retrieve the PubMed articles for this cwid if the articles have not been retrieved yet. 
			PubmedXmlFetcher pubmedXmlFetcher = new PubmedXmlFetcher();
			pubmedXmlFetcher.setPerformRetrievePublication(reCiterConfigProperty.isPerformRetrievePublication());
			List<PubmedArticle> pubmedArticleList = pubmedXmlFetcher.getPubmedArticle(lastName, firstInitial, cwid);

			// Retrieve the scopus affiliation information for this cwid if the affiliations have not been retrieve yet.
			ScopusXmlFetcher scopusXmlFetcher = new ScopusXmlFetcher();
			List<ScopusEntry> scopusEntryList = scopusXmlFetcher.getScopusEntryList(lastName, firstInitial, cwid);

			// Map the pmid to a ScopusEntry.
			Map<String, ScopusEntry> pmidToScopusEntry = new HashMap<String, ScopusEntry>();
			for (ScopusEntry entry : scopusEntryList) {
				pmidToScopusEntry.put(entry.getPubmedID(), entry);
			}

			// Need to integrate the Scopus information into PubmedArticle. Add a fake author which contains the
			// Scopus Affiliation. The fake author has pmid as last name and first name.
			for (PubmedArticle pubmedArticle : pubmedArticleList) {
				String pmid = pubmedArticle.getMedlineCitation().getPmid().getPmidString();

				if (pmidToScopusEntry.containsKey(pmid)) {
					String scopusAffiliation = pmidToScopusEntry.get(pmid).affiliationConcatForm();
					MedlineCitationArticleAuthor fakeAuthor = new MedlineCitationArticleAuthor();
					fakeAuthor.setLastName(pmid);
					fakeAuthor.setForeName(pmid);
					fakeAuthor.setAffiliation(scopusAffiliation);
					if (pubmedArticle.getMedlineCitation().getArticle().getAuthorList() == null) {
						pubmedArticle.getMedlineCitation().getArticle().setAuthorList(new ArrayList<MedlineCitationArticleAuthor>());
					}
					pubmedArticle.getMedlineCitation().getArticle().getAuthorList().add(fakeAuthor);
				}
			}

			// Convert PubmedArticle to ReCiterArticle.
			List<ReCiterArticle> reCiterArticleList = ArticleTranslator.translateAll(pubmedArticleList);

			// Add TargetAuthor Article:
			reCiterArticleList.add(targetAuthorArticle);

			// Convert ReCiterArticle to Lucene Document
			List<Document> luceneDocumentList = DocumentTranslator.translateAll(reCiterArticleList);

			// If Lucene index already exist for this cwid, read from index. Else write to index.
			// Use Lucene to write to index.
			DocumentIndexWriter docIndexWriter = new DocumentIndexWriter(cwid);
			docIndexWriter.indexAll(luceneDocumentList);<commit_msg>Commented out the section where ReCiterExample tries to fetch Scopus data.<commit_after>package test.examples.pubmed;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import main.database.dao.ArticleDao;
import main.reciter.algorithm.cluster.ReCiterClusterer;
import main.reciter.lucene.DocumentIndexReader;
import main.reciter.lucene.DocumentIndexWriter;
import main.reciter.lucene.DocumentTranslator;
import main.reciter.model.article.ReCiterArticle;
import main.reciter.model.article.ReCiterArticleCoAuthors;
import main.reciter.model.article.ReCiterArticleKeywords;
import main.reciter.model.author.AuthorAffiliation;
import main.reciter.model.author.AuthorName;
import main.reciter.model.author.ReCiterAuthor;
import main.reciter.model.author.TargetAuthor;
import main.reciter.utils.Analysis;
import main.reciter.utils.AnalysisCSVWriter;
import main.reciter.utils.AnalysisObject;
import main.reciter.utils.ReCiterConfigProperty;
import main.xml.pubmed.PubmedXmlFetcher;
import main.xml.pubmed.model.MedlineCitationArticleAuthor;
import main.xml.pubmed.model.PubmedArticle;
import main.xml.scopus.ScopusXmlFetcher;
import main.xml.scopus.model.ScopusEntry;
import main.xml.translator.ArticleTranslator;

import org.apache.lucene.document.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ReCiterExample {

	private final static Logger slf4jLogger = LoggerFactory.getLogger(ReCiterExample.class);

	public static void main(String[] args) throws IOException {
		// Read Property file from config.properties.
		ReCiterConfigProperty reCiterConfigProperty = new ReCiterConfigProperty();
		reCiterConfigProperty.loadProperty("config.properties");

		// Keep track of exeuction time of ReCiter.
		long startTime = System.currentTimeMillis();

		// Run ReCiter.
		ReCiterExample reCiterExample = new ReCiterExample();
		reCiterExample.runExample(reCiterConfigProperty);

		long stopTime = System.currentTimeMillis();
		long elapsedTime = stopTime - startTime;
		slf4jLogger.info("Total execution time: " + elapsedTime + " ms.");
	}

	/**
	 * Setup the data to run the ReCiter algorithm.
	 * @param lastName
	 * @param firstInitial
	 * @param cwid
	 */
	public void runExample(ReCiterConfigProperty reCiterConfigProperty) {

		String lastName = reCiterConfigProperty.getLastName();
		String middleName = reCiterConfigProperty.getMiddleName();
		String firstName = reCiterConfigProperty.getFirstName();
		String affiliation = reCiterConfigProperty.getAuthorAffiliation();
		String firstInitial = firstName.substring(0, 1);
		String cwid = reCiterConfigProperty.getCwid();

		String authorKeywords = reCiterConfigProperty.getAuthorKeywords();
		String coAuthors = reCiterConfigProperty.getCoAuthors();
		double similarityThreshold = reCiterConfigProperty.getSimilarityThreshold();

		String department = reCiterConfigProperty.getAuthorDepartment();
		
		DocumentIndexWriter.setUseStopWords(reCiterConfigProperty.isUseStemming()); // revise this logic.
		
		// Define Singleton target author.
		TargetAuthor.init(new AuthorName(firstName, middleName, lastName), new AuthorAffiliation(affiliation));
		ReCiterArticle targetAuthorArticle = new ReCiterArticle(-1);
		targetAuthorArticle.setArticleCoAuthors(new ReCiterArticleCoAuthors());
		targetAuthorArticle.getArticleCoAuthors().addCoAuthor(new ReCiterAuthor(new AuthorName(firstName, middleName, lastName), new AuthorAffiliation(affiliation + " " + department)));		

		targetAuthorArticle.setArticleKeywords(new ReCiterArticleKeywords());
		for (String keyword : authorKeywords.split(",")) {
			targetAuthorArticle.getArticleKeywords().addKeyword(keyword);
		}

		for (String author : coAuthors.split(",")) {
			String[] authorArray = author.split(" ");

			if (authorArray.length == 2) {
				String coAuthorFirstName = authorArray[0];
				String coAuthorLastName = authorArray[1];
				targetAuthorArticle.getArticleCoAuthors().addCoAuthor(new ReCiterAuthor(new AuthorName(coAuthorFirstName, "", coAuthorLastName), new AuthorAffiliation("")));
			} else if (authorArray.length == 3) {
				String coAuthorFirstName = authorArray[0];
				String coAuthorMiddleName = authorArray[1];
				String coAuthorLastName = authorArray[2];
				targetAuthorArticle.getArticleCoAuthors().addCoAuthor(new ReCiterAuthor(new AuthorName(coAuthorFirstName, coAuthorMiddleName, coAuthorLastName), new AuthorAffiliation("")));
			}
		}
		
		
		// Try reading from Lucene Index:
		DocumentIndexReader documentIndexReader = new DocumentIndexReader();

		// Lucene Index doesn't contain this cwid's files.
		if (!documentIndexReader.isIndexed(cwid)) {

			slf4jLogger.debug("index...");

			// Retrieve the PubMed articles for this cwid if the articles have not been retrieved yet. 
			PubmedXmlFetcher pubmedXmlFetcher = new PubmedXmlFetcher();
			pubmedXmlFetcher.setPerformRetrievePublication(reCiterConfigProperty.isPerformRetrievePublication());
			List<PubmedArticle> pubmedArticleList = pubmedXmlFetcher.getPubmedArticle(lastName, firstInitial, cwid);

			// Retrieve the scopus affiliation information for this cwid if the affiliations have not been retrieve yet.
//			ScopusXmlFetcher scopusXmlFetcher = new ScopusXmlFetcher();
//			List<ScopusEntry> scopusEntryList = scopusXmlFetcher.getScopusEntryList(lastName, firstInitial, cwid);
//
//			// Map the pmid to a ScopusEntry.
//			Map<String, ScopusEntry> pmidToScopusEntry = new HashMap<String, ScopusEntry>();
//			for (ScopusEntry entry : scopusEntryList) {
//				pmidToScopusEntry.put(entry.getPubmedID(), entry);
//			}
//
//			// Need to integrate the Scopus information into PubmedArticle. Add a fake author which contains the
//			// Scopus Affiliation. The fake author has pmid as last name and first name.
//			for (PubmedArticle pubmedArticle : pubmedArticleList) {
//				String pmid = pubmedArticle.getMedlineCitation().getPmid().getPmidString();
//
//				if (pmidToScopusEntry.containsKey(pmid)) {
//					String scopusAffiliation = pmidToScopusEntry.get(pmid).affiliationConcatForm();
//					MedlineCitationArticleAuthor fakeAuthor = new MedlineCitationArticleAuthor();
//					fakeAuthor.setLastName(pmid);
//					fakeAuthor.setForeName(pmid);
//					fakeAuthor.setAffiliation(scopusAffiliation);
//					if (pubmedArticle.getMedlineCitation().getArticle().getAuthorList() == null) {
//						pubmedArticle.getMedlineCitation().getArticle().setAuthorList(new ArrayList<MedlineCitationArticleAuthor>());
//					}
//					pubmedArticle.getMedlineCitation().getArticle().getAuthorList().add(fakeAuthor);
//				}
//			}

			// Convert PubmedArticle to ReCiterArticle.
			List<ReCiterArticle> reCiterArticleList = ArticleTranslator.translateAll(pubmedArticleList);

			// Add TargetAuthor Article:
			reCiterArticleList.add(targetAuthorArticle);

			// Convert ReCiterArticle to Lucene Document
			List<Document> luceneDocumentList = DocumentTranslator.translateAll(reCiterArticleList);

			// If Lucene index already exist for this cwid, read from index. Else write to index.
			// Use Lucene to write to index.
			DocumentIndexWriter docIndexWriter = new DocumentIndexWriter(cwid);
			docIndexWriter.indexAll(luceneDocumentList);
{'fraction_non_alphanumeric': 0.09678984954586424, 'fraction_numerical': 0.0020800110933924983, 'mean_word_length': 5.577291381668947, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7650075', 'n_tokens_mistral': 4763, 'n_tokens_neox': 4182, 'n_words': 870}