<commit_before>package edu.wustl.catissuecore.deid;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Date;
import java.util.List;

import org.jdom.Element;
import org.jdom.output.Format;

import edu.wustl.catissuecore.caties.util.CSVLogger;
import edu.wustl.catissuecore.caties.util.CaCoreAPIService;
import edu.wustl.catissuecore.caties.util.CaTIESConstants;
import edu.wustl.catissuecore.caties.util.CaTIESProperties;
import edu.wustl.catissuecore.domain.Participant;
import edu.wustl.catissuecore.domain.pathology.DeidentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.IdentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.TextContent;
import edu.wustl.catissuecore.util.global.Constants;
import edu.wustl.common.util.logger.Logger;

/**
 * This class is a thread which converts a single identified report into its equivalent de-identified report.
 * @author vijay_pande
 */
public class DeidReportThread extends Thread
{
	public static final Object obj=new Object();
	private IdentifiedSurgicalPathologyReport identifiedReport;
	
	/**
	 * constructor for the DeidReportThread thread 
	 * @param identifiedReport identified Surgical Pathology Report
	 * @throws Exception Generic excpetion
	 */
	public DeidReportThread(IdentifiedSurgicalPathologyReport ispr) throws Exception
	{
		this.identifiedReport=ispr;	
	}
	
	/**
	 * This is default run method of the thread. Which is like a deid pipeline. This pipeline manages the de-identification process. 
	 * @see java.lang.Thread#run()
	 */
	public void run()
	{
		Long startTime = new Date().getTime();
		try
		{
			Logger.out.info("De-identification process started for "+identifiedReport.getId().toString());
			// instantiate document
			org.jdom.Document currentRequestDocument = new org.jdom.Document(new Element("Dataset"));
			// get participant from CPR
			String hqlQuery="select cpr.participant from edu.wustl.catissuecore.domain.CollectionProtocolRegistration cpr, " +
					" edu.wustl.catissuecore.domain.SpecimenCollectionGroup scg" +
					" where scg.id="+identifiedReport.getSpecimenCollectionGroup().getId() +
					" and scg.id in elements(cpr.specimenCollectionGroupCollection)";
			List participantList=(List)CaCoreAPIService.executeQuery(hqlQuery, Participant.class.getName());
			Participant participant=null;
			if(participantList!=null && participantList.size()>0)
			{
				participant=(Participant)participantList.get(0);
			}
			
			// get textcontent
			TextContent textContent=(TextContent)CaCoreAPIService.getObject(TextContent.class, Constants.SYSTEM_IDENTIFIER, identifiedReport.getTextContent().getId());
			
			// build report element using report text
			Element reportElement = DeidUtils.buildReportElement(participant, identifiedReport, textContent.getData());
			// add report element to root of the document
			currentRequestDocument.getRootElement().addContent(reportElement);
	        
			// convert document ino string
	        String deidRequest = DeidUtils.convertDocumentToString(currentRequestDocument, Format.getPrettyFormat()); 
	        Logger.out.info("Calling native call for report "+identifiedReport.getId().toString());
	        // function call which contains the actual native call for deidentification
	        String deidReportText=deIdentify(deidRequest);
	        Logger.out.info("Calling native finished successfully for report "+identifiedReport.getId().toString());
	        String deidText="";
	        Logger.out.info("Extracting report text for report "+identifiedReport.getId().toString());
	        // extract the report text
	        deidText=DeidUtils.extractReport(deidReportText, CaTIESProperties.getValue(CaTIESConstants.DEID_DTD_FILENAME));
	        Logger.out.info("Extracting report text finished for report "+identifiedReport.getId().toString());
	        Date deidCollectionDate=null;
	   
	        // ectract collection date and time
	        deidCollectionDate=DeidUtils.extractDate(deidText);	
	        deidText = deidText.substring(0, deidText.lastIndexOf("||-"));
	        Logger.out.info("Creating deidentified report for identified report id="+identifiedReport.getId().toString());
	        // Create object of deidentified report
	        DeidentifiedSurgicalPathologyReport pathologyReport = createDeidPathologyReport(identifiedReport, deidText, deidCollectionDate);
	        Logger.out.info("De-identification process finished for "+identifiedReport.getId().toString());
	        saveReports(pathologyReport);
	        Long endTime = new Date().getTime();
	        CSVLogger.info(CaTIESConstants.LOGGER_DEID_SERVER, new Date().toString()+","+identifiedReport.getId()+","+CaTIESConstants.DEIDENTIFIED+","+"Report De-identified successfully,"+(endTime-startTime));
		}
    	catch(Throwable ex)
    	{
    		Logger.out.error("Deidentification process is failed:",ex);
			try
			{
				Long endTime = new Date().getTime();
				CSVLogger.error(CaTIESConstants.LOGGER_DEID_SERVER, new Date().toString()+","+identifiedReport.getId()+","+CaTIESConstants.FAILURE+","+ex.getMessage()+",,"+(endTime-startTime));
				// if any exception occures then update the status of the identified report to failed
				identifiedReport.setReportStatus(CaTIESConstants.DEID_PROCESS_FAILED);
				CaCoreAPIService.updateObject(identifiedReport);
			}
			catch(Exception e)
			{
				Logger.out.error("DeidReportThread: Updating Identified report status failed",e);
			}
    		Logger.out.error("Upexpected error in DeidReportThread thread", ex);
    	}
	}
	
	/**
	 * Method to create and initialize object of DeidentifiedSurgicalPathologyReport
	 * @param identifiedReport identified surgical pathology report
	 * @param deidText de-intified text
	 * @param deidCollectedDate collection date and time of report
	 * @return DeidentifiedSurgicalPathologyReport
	 * @throws Exception a generic exception oocured while creating de-identified report instance.
	 */
	private DeidentifiedSurgicalPathologyReport createDeidPathologyReport(IdentifiedSurgicalPathologyReport ispr, String deidText,
            Date deidCollectedDate) throws Exception
	{
		Logger.out.info("Creating deid report for identifiedReport id="+ispr.getId());
		// instnatiate deidentified report
		DeidentifiedSurgicalPathologyReport deidReport=new DeidentifiedSurgicalPathologyReport();
		
		if (ispr.getCollectionDateTime() != null) 
		{
			//if collection date and time is not null then set it to deidentifide repot
            deidReport.setCollectionDateTime(deidCollectedDate);
        }
		  
		// set default values for deidentified report
		deidReport.setActivityStatus(ispr.getActivityStatus()); 
       	deidReport.setReportStatus(CaTIESConstants.PENDING_FOR_XML);
       	deidReport.setIsQuanrantined(Constants.ACTIVITY_STATUS_ACTIVE);
        deidReport.setSpecimenCollectionGroup(ispr.getSpecimenCollectionGroup());
        TextContent tc=new TextContent();
        // set deidentified text to text content
        tc.setData(deidText);
        // set identified report to deidentified report
        tc.setSurgicalPathologyReport(deidReport);
        
        // set reportSource for deidentified report
        deidReport.setReportSource(ispr.getReportSource());
        // set text content which contains deidentified text to deidentified report 
        deidReport.setTextContent(tc);
        // set default value for flag for review
        deidReport.setIsFlagForReview(new Boolean(false));
        
        return deidReport;
    }
	
	/**
	 * This method is responsible for preparing and calling a native method call to convert plain text into deindentified text.
	 * @param text text to be de-identified
	 * @return de-identified text
	 * @throws Exception ocured while calling a native method call for de-identification
	 */
	public String deIdentify(String text) throws Exception
	{
		
		String output = "";	
		try 
		{
			synchronized(obj)
			{
				// file for xml formated input text
				File predeidFile = new File("predeid.xml");
				// temp file for processing input file
				File postdeidFile = new File("postdeid.tmp");<commit_msg>setter for collectionDateTime and reportsource removed<commit_after>package edu.wustl.catissuecore.deid;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Date;
import java.util.List;

import org.jdom.Element;
import org.jdom.output.Format;

import edu.wustl.catissuecore.caties.util.CSVLogger;
import edu.wustl.catissuecore.caties.util.CaCoreAPIService;
import edu.wustl.catissuecore.caties.util.CaTIESConstants;
import edu.wustl.catissuecore.caties.util.CaTIESProperties;
import edu.wustl.catissuecore.domain.Participant;
import edu.wustl.catissuecore.domain.pathology.DeidentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.IdentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.TextContent;
import edu.wustl.catissuecore.util.global.Constants;
import edu.wustl.common.util.logger.Logger;

/**
 * This class is a thread which converts a single identified report into its equivalent de-identified report.
 * @author vijay_pande
 */
public class DeidReportThread extends Thread
{
	public static final Object obj=new Object();
	private IdentifiedSurgicalPathologyReport identifiedReport;
	
	/**
	 * constructor for the DeidReportThread thread 
	 * @param identifiedReport identified Surgical Pathology Report
	 * @throws Exception Generic excpetion
	 */
	public DeidReportThread(IdentifiedSurgicalPathologyReport ispr) throws Exception
	{
		this.identifiedReport=ispr;	
	}
	
	/**
	 * This is default run method of the thread. Which is like a deid pipeline. This pipeline manages the de-identification process. 
	 * @see java.lang.Thread#run()
	 */
	public void run()
	{
		Long startTime = new Date().getTime();
		try
		{
			Logger.out.info("De-identification process started for "+identifiedReport.getId().toString());
			// instantiate document
			org.jdom.Document currentRequestDocument = new org.jdom.Document(new Element("Dataset"));
			// get participant from CPR
			String hqlQuery="select cpr.participant from edu.wustl.catissuecore.domain.CollectionProtocolRegistration cpr, " +
					" edu.wustl.catissuecore.domain.SpecimenCollectionGroup scg" +
					" where scg.id="+identifiedReport.getSpecimenCollectionGroup().getId() +
					" and scg.id in elements(cpr.specimenCollectionGroupCollection)";
			List participantList=(List)CaCoreAPIService.executeQuery(hqlQuery, Participant.class.getName());
			Participant participant=null;
			if(participantList!=null && participantList.size()>0)
			{
				participant=(Participant)participantList.get(0);
			}
			
			// get textcontent
			TextContent textContent=(TextContent)CaCoreAPIService.getObject(TextContent.class, Constants.SYSTEM_IDENTIFIER, identifiedReport.getTextContent().getId());
			
			// build report element using report text
			Element reportElement = DeidUtils.buildReportElement(participant, identifiedReport, textContent.getData());
			// add report element to root of the document
			currentRequestDocument.getRootElement().addContent(reportElement);
	        
			// convert document ino string
	        String deidRequest = DeidUtils.convertDocumentToString(currentRequestDocument, Format.getPrettyFormat()); 
	        Logger.out.info("Calling native call for report "+identifiedReport.getId().toString());
	        // function call which contains the actual native call for deidentification
	        String deidReportText=deIdentify(deidRequest);
	        Logger.out.info("Calling native finished successfully for report "+identifiedReport.getId().toString());
	        String deidText="";
	        Logger.out.info("Extracting report text for report "+identifiedReport.getId().toString());
	        // extract the report text
	        deidText=DeidUtils.extractReport(deidReportText, CaTIESProperties.getValue(CaTIESConstants.DEID_DTD_FILENAME));
	        Logger.out.info("Extracting report text finished for report "+identifiedReport.getId().toString());
	        Date deidCollectionDate=null;
	   
	        // ectract collection date and time
	        deidCollectionDate=DeidUtils.extractDate(deidText);	
	        deidText = deidText.substring(0, deidText.lastIndexOf("||-"));
	        Logger.out.info("Creating deidentified report for identified report id="+identifiedReport.getId().toString());
	        // Create object of deidentified report
	        DeidentifiedSurgicalPathologyReport pathologyReport = createDeidPathologyReport(identifiedReport, deidText, deidCollectionDate);
	        Logger.out.info("De-identification process finished for "+identifiedReport.getId().toString());
	        saveReports(pathologyReport);
	        Long endTime = new Date().getTime();
	        CSVLogger.info(CaTIESConstants.LOGGER_DEID_SERVER, new Date().toString()+","+identifiedReport.getId()+","+CaTIESConstants.DEIDENTIFIED+","+"Report De-identified successfully,"+(endTime-startTime));
		}
    	catch(Throwable ex)
    	{
    		Logger.out.error("Deidentification process is failed:",ex);
			try
			{
				Long endTime = new Date().getTime();
				CSVLogger.error(CaTIESConstants.LOGGER_DEID_SERVER, new Date().toString()+","+identifiedReport.getId()+","+CaTIESConstants.FAILURE+","+ex.getMessage()+",,"+(endTime-startTime));
				// if any exception occures then update the status of the identified report to failed
				identifiedReport.setReportStatus(CaTIESConstants.DEID_PROCESS_FAILED);
				CaCoreAPIService.updateObject(identifiedReport);
			}
			catch(Exception e)
			{
				Logger.out.error("DeidReportThread: Updating Identified report status failed",e);
			}
    		Logger.out.error("Upexpected error in DeidReportThread thread", ex);
    	}
	}
	
	/**
	 * Method to create and initialize object of DeidentifiedSurgicalPathologyReport
	 * @param identifiedReport identified surgical pathology report
	 * @param deidText de-intified text
	 * @param deidCollectedDate collection date and time of report
	 * @return DeidentifiedSurgicalPathologyReport
	 * @throws Exception a generic exception oocured while creating de-identified report instance.
	 */
	private DeidentifiedSurgicalPathologyReport createDeidPathologyReport(IdentifiedSurgicalPathologyReport ispr, String deidText,
            Date deidCollectedDate) throws Exception
	{
		Logger.out.info("Creating deid report for identifiedReport id="+ispr.getId());
		// instnatiate deidentified report
		DeidentifiedSurgicalPathologyReport deidReport=new DeidentifiedSurgicalPathologyReport();
        		  
		// set default values for deidentified report
		deidReport.setActivityStatus(ispr.getActivityStatus()); 
       	deidReport.setReportStatus(CaTIESConstants.PENDING_FOR_XML);
       	deidReport.setIsQuanrantined(Constants.ACTIVITY_STATUS_ACTIVE);
        deidReport.setSpecimenCollectionGroup(ispr.getSpecimenCollectionGroup());
        TextContent tc=new TextContent();
        // set deidentified text to text content
        tc.setData(deidText);
        // set identified report to deidentified report
        tc.setSurgicalPathologyReport(deidReport);
        
        // set text content which contains deidentified text to deidentified report 
        deidReport.setTextContent(tc);
        // set default value for flag for review
        deidReport.setIsFlagForReview(new Boolean(false));
        
        return deidReport;
    }
	
	/**
	 * This method is responsible for preparing and calling a native method call to convert plain text into deindentified text.
	 * @param text text to be de-identified
	 * @return de-identified text
	 * @throws Exception ocured while calling a native method call for de-identification
	 */
	public String deIdentify(String text) throws Exception
	{
		
		String output = "";	
		try 
		{
			synchronized(obj)
			{
				// file for xml formated input text
				File predeidFile = new File("predeid.xml");
				// temp file for processing input file
				File postdeidFile = new File("postdeid.tmp");
{'fraction_non_alphanumeric': 0.09280972765059867, 'fraction_numerical': 0.0003722315280104225, 'mean_word_length': 4.328925619834711, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7319400', 'n_tokens_mistral': 4871, 'n_tokens_neox': 4416, 'n_words': 1042}