<commit_before>package edu.wustl.catissuecore.deidentifier.harvardscrubber;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;

import org.jdom.CDATA;
import org.jdom.DocType;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.output.Format;
import org.spin.loader.scrubber.impl.Scrubber;
import org.spin.loader.scrubber.impl.ScrubberBuilder;
import org.spin.loader.scrubber.tools.JDOMTools;

import edu.wustl.catissuecore.caties.util.CaTIESConstants;
import edu.wustl.catissuecore.caties.util.CaTIESProperties;
import edu.wustl.catissuecore.caties.util.Utility;
import edu.wustl.catissuecore.deidentifier.AbstractDeidentifier;
import edu.wustl.catissuecore.domain.Participant;
import edu.wustl.catissuecore.domain.ParticipantMedicalIdentifier;
import edu.wustl.catissuecore.domain.pathology.DeidentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.IdentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.TextContent;
import edu.wustl.catissuecore.util.global.Constants;

public class HarvardSrubberDeidentifier extends AbstractDeidentifier 
{
	private static Scrubber scrubber;
	public DeidentifiedSurgicalPathologyReport deidentify(
			IdentifiedSurgicalPathologyReport identifiedReport) throws Exception 
	{
		File fileTosScrub=getFileToScrub(identifiedReport);
		String scrubbed=scrubber.scrub(fileTosScrub);
		fileTosScrub.delete();
		
		String deidReportText=Utility.extractReport(scrubbed, CaTIESProperties.getValue(CaTIESConstants.HARVARD_SCRUBBER_DTD_FILENAME), CaTIESConstants.HARVARD_SCRUBBER_XPATH, CaTIESConstants.TAG_FULL_REPORT_TEXT);
		
		DeidentifiedSurgicalPathologyReport deidentifiedSurgicalPathologyReport=new DeidentifiedSurgicalPathologyReport();
		TextContent textContent=new TextContent();
		textContent.setData(deidReportText);
		textContent.setSurgicalPathologyReport(deidentifiedSurgicalPathologyReport);
		
		deidentifiedSurgicalPathologyReport.setTextContent(textContent);
		return deidentifiedSurgicalPathologyReport;
	}

	private synchronized File getFileToScrub(IdentifiedSurgicalPathologyReport identifiedReport) throws Exception
	{
		org.jdom.Document currentRequestDocument = new org.jdom.Document(new Element(CaTIESConstants.TAG_ENVELOPE));
		Element reportHeader=buildReportHeader(identifiedReport);
		currentRequestDocument.getRootElement().addContent(reportHeader);
		
		DocType docType=new DocType("HarvardScrubber","file:///"+CaTIESProperties.getValue(CaTIESConstants.HARVARD_SCRUBBER_DTD_FILENAME));
		
		currentRequestDocument.setDocType(docType);<commit_msg>Changes for synchronization<commit_after>package edu.wustl.catissuecore.deidentifier.harvardscrubber;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;

import org.jdom.CDATA;
import org.jdom.DocType;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.output.Format;
import org.spin.loader.scrubber.impl.Scrubber;
import org.spin.loader.scrubber.impl.ScrubberBuilder;
import org.spin.loader.scrubber.tools.JDOMTools;

import edu.wustl.catissuecore.caties.util.CaTIESConstants;
import edu.wustl.catissuecore.caties.util.CaTIESProperties;
import edu.wustl.catissuecore.caties.util.Utility;
import edu.wustl.catissuecore.deidentifier.AbstractDeidentifier;
import edu.wustl.catissuecore.domain.Participant;
import edu.wustl.catissuecore.domain.ParticipantMedicalIdentifier;
import edu.wustl.catissuecore.domain.pathology.DeidentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.IdentifiedSurgicalPathologyReport;
import edu.wustl.catissuecore.domain.pathology.TextContent;
import edu.wustl.catissuecore.util.global.Constants;

public class HarvardSrubberDeidentifier extends AbstractDeidentifier 
{
	private static Scrubber scrubber;
	public DeidentifiedSurgicalPathologyReport deidentify(
			IdentifiedSurgicalPathologyReport identifiedReport) throws Exception 
	{
		String scrubbed;
		synchronized (scrubber) 
		{
			File fileTosScrub=getFileToScrub(identifiedReport);
			scrubbed=scrubber.scrub(fileTosScrub);
			fileTosScrub.delete();
		}
		
		
		String deidReportText=Utility.extractReport(scrubbed, CaTIESProperties.getValue(CaTIESConstants.HARVARD_SCRUBBER_DTD_FILENAME), CaTIESConstants.HARVARD_SCRUBBER_XPATH, CaTIESConstants.TAG_FULL_REPORT_TEXT);
		
		DeidentifiedSurgicalPathologyReport deidentifiedSurgicalPathologyReport=new DeidentifiedSurgicalPathologyReport();
		TextContent textContent=new TextContent();
		textContent.setData(deidReportText);
		textContent.setSurgicalPathologyReport(deidentifiedSurgicalPathologyReport);
		
		deidentifiedSurgicalPathologyReport.setTextContent(textContent);
		return deidentifiedSurgicalPathologyReport;
	}

	private File getFileToScrub(IdentifiedSurgicalPathologyReport identifiedReport) throws Exception
	{
		org.jdom.Document currentRequestDocument = new org.jdom.Document(new Element(CaTIESConstants.TAG_ENVELOPE));
		Element reportHeader=buildReportHeader(identifiedReport);
		currentRequestDocument.getRootElement().addContent(reportHeader);
		
		DocType docType=new DocType("HarvardScrubber","file:///"+CaTIESProperties.getValue(CaTIESConstants.HARVARD_SCRUBBER_DTD_FILENAME));
		
		currentRequestDocument.setDocType(docType);
{'fraction_non_alphanumeric': 0.0818576075171666, 'fraction_numerical': 0.0, 'mean_word_length': 8.163907284768213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4828687', 'n_tokens_mistral': 1866, 'n_tokens_neox': 1736, 'n_words': 223}