<commit_before>package gov.nih.nci.cabig.caaers.web.admin;

import gov.nih.nci.cabig.caaers.CaaersSystemException;
import gov.nih.nci.cabig.caaers.dao.AgentDao;
import gov.nih.nci.cabig.caaers.dao.AgentSpecificTermDao;
import gov.nih.nci.cabig.caaers.dao.StudyAgentDao;
import gov.nih.nci.cabig.caaers.dao.StudyDao;
import gov.nih.nci.cabig.caaers.domain.*;
import gov.nih.nci.cabig.caaers.domain.repository.TerminologyRepository;
import gov.nih.nci.cabig.caaers.service.AgentSpecificAdverseEventListService;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;

import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/*
* @author <NAME>
* 
* */
public class AgentSpecificTermsImporter {

    private static final String KEY_MISSING_TERMS = "missingTerms";
    private static final String KEY_PROCESSED_AGENTS = "processedAgents";
    private static final String KEY_PROCESSED_AGENTTERMS = "processedAgentTerms";

	private POIFSFileSystem poifs;
	private HSSFWorkbook wb;
	private HSSFSheet sh;

	private AgentDao agentDao;
    private TerminologyRepository terminologyRepository;
    private AgentSpecificTermDao agentSpecificTermDao;
    private AgentSpecificAdverseEventListService asaelService;
    private StudyAgentDao studyAgentDao;
    private StudyDao studyDao;

    private int rowCount = 0;
    private int columnsCount = 0;
    private Map<String, Agent> agents = new HashMap<String, Agent>();
    private List<String> missingTerms = new ArrayList<String>();
    private Map<String, String> asaelCache = new HashMap<String, String>();
    private int asael;
    
    private Map<String, Short> headers = new HashMap() {{
        put("NSC", -1);
        put("CTCAE_VERSION", -1);
        put("CTCAE_CATEGORY", -1);
        put("AE_TERM", -1);
    }};

    public static File file = new File("/home/dell/Desktop/asael.xls");

    public AgentSpecificTermsImporter() {
    }
    
    public AgentSpecificTermsImporter(File f) {
        file = f;
    }

    public static void main(String[] args) throws Exception {
        AgentSpecificTermsImporter im = new AgentSpecificTermsImporter(file);
        im.importFile();
    }

    public Map<String, Object> importFile() throws Exception {
        Map<String, Object> results = new HashMap<String, Object>();
        
        bootstrap(file);
        agents.clear();
        missingTerms.clear();
        asael = 0;

        // Loading ASAE list
        int i = 1;
        while (i <= rowCount) {

            HSSFRow row = sh.getRow(i);
            if (row != null) {
                String nsc = getCellData("", i, row.getCell((short)headers.get("NSC")));
                String ctcae_category= getCellData("", i, row.getCell((short)headers.get("CTCAE_CATEGORY")));
                int ctcae_version = Integer.parseInt(getCellData("", i, row.getCell((short)headers.get("CTCAE_VERSION"))));
                String ae_term = getCellData("", i, row.getCell((short)headers.get("AE_TERM")));

                // System.out.println(String.format("NSC:%s,   V:%s,   C:%s,   T:%s", nsc, ctcae_version, ctcae_category, ae_term));

                Agent a = agents.get(nsc);
                if (a == null) {
                    a = agentDao.getByNscNumber(nsc);
                    // System.out.println(asael + ". OK. Found agent [" + a.getName() + "] for NSC: [" + nsc + "]");
                    agents.put(nsc, a);
                }
                
                if (a != null) {
                    AgentSpecificCtcTerm t = new AgentSpecificCtcTerm();
                    t.setAgent(a);

                    List<CtcTerm> list = terminologyRepository.getCtcTerm(ctcae_category, ctcae_version, ae_term);
                    if (list.size() == 0) {
                        // System.out.println("Err. Term not found: " + ae_term);
                        missingTerms.add(ae_term);
                    } else {
                        t.setCtcTerm(list.get(0));
                        if (persistASAE(t)) asael++;
                    }
                    
                } else {
                    // System.out.println("Err. The agent was not found by its NSC: " + nsc);
                }

            }
            i++;
        }

        results.put(KEY_MISSING_TERMS, missingTerms);
        results.put(KEY_PROCESSED_AGENTS, agents.size());
        results.put(KEY_PROCESSED_AGENTTERMS, asael);

/*
        System.out.println();
        System.out.println(String.format("Loaded %d agents", agents.size()));
        System.out.println(String.format("Built %d ASAE terms", asael));
        System.out.println("OK.");
*/

        return results;
    }

    private boolean isAgentSpecificTermPersisted(AgentSpecificTerm at) {
        List<AgentSpecificCtcTerm> l = agentSpecificTermDao.getCtcTerm(at.getAgent().getId(), at.getTerm().getId());
        if (l.size() > 0) return true;
        return false;
    }

    /*
    * Persisting ASAE list
    * */
    private boolean persistASAE(AgentSpecificTerm t) {
        if (!isAgentSpecificTermPersisted(t)) {
            agentSpecificTermDao.save(t);
            List<StudyAgent> l = getStudyAgentDao().getByAgentID(t.getAgent().getId());
            for (StudyAgent s : l) {
                asaelService.synchronizeStudyWithAgentTerm(s.getStudy(), t);
                studyDao.save(s.getStudy());
            }
            return true;
        }
        return false;
    }

    private void bootstrap(File inputFile) throws Exception {
        poifs = new POIFSFileSystem(new FileInputStream(inputFile));

        // create a workbook out of the input stream
        wb = new HSSFWorkbook(poifs);
        sh = wb.getSheetAt(0);
        rowCount = sh.getLastRowNum();
        columnsCount = sh.getRow(0).getLastCellNum();

        // get needed headers
        for (byte i = 0; i<columnsCount; i++) {
            HSSFCell cell = sh.getRow(0).getCell(i);
            if (headers.containsKey(cell.getStringCellValue())) {<commit_msg>CAAERS-3877 - The system must support the import and usage of the Agent Specific Adverse Event List (ASAEL)<commit_after>package gov.nih.nci.cabig.caaers.web.admin;

import gov.nih.nci.cabig.caaers.CaaersSystemException;
import gov.nih.nci.cabig.caaers.dao.AgentDao;
import gov.nih.nci.cabig.caaers.dao.AgentSpecificTermDao;
import gov.nih.nci.cabig.caaers.dao.StudyAgentDao;
import gov.nih.nci.cabig.caaers.dao.StudyDao;
import gov.nih.nci.cabig.caaers.domain.*;
import gov.nih.nci.cabig.caaers.domain.repository.TerminologyRepository;
import gov.nih.nci.cabig.caaers.service.AgentSpecificAdverseEventListService;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/*
* @author <NAME>
* 
* */
public class AgentSpecificTermsImporter {

    private static final String KEY_MISSING_TERMS = "missingTerms";
    private static final String KEY_PROCESSED_AGENTS = "processedAgents";
    private static final String KEY_PROCESSED_AGENTTERMS = "processedAgentTerms";

	private POIFSFileSystem poifs;
	private HSSFWorkbook wb;
	private HSSFSheet sh;

	private AgentDao agentDao;
    private TerminologyRepository terminologyRepository;
    private AgentSpecificTermDao agentSpecificTermDao;
    private AgentSpecificAdverseEventListService asaelService;
    private StudyAgentDao studyAgentDao;
    private StudyDao studyDao;

    private int rowCount = 0;
    private int columnsCount = 0;
    private Map<String, Agent> agents = new HashMap<String, Agent>();
    private List<String> missingTerms = new ArrayList<String>();
    private Map<String, String> asaelCache = new HashMap<String, String>();
    private int asael;
    
    private Map<String, Short> headers = new HashMap() {{
        put("NSC", -1);
        put("CTCAE_VERSION", -1);
        put("CTCAE_CATEGORY", -1);
        put("AE_TERM", -1);
    }};

    public static File file = new File("/home/dell/Desktop/asael.xls");

    public AgentSpecificTermsImporter() {
    }
    
    public AgentSpecificTermsImporter(File f) {
        file = f;
    }

    public static void main(String[] args) throws Exception {
        AgentSpecificTermsImporter im = new AgentSpecificTermsImporter(file);
        im.importFile();
    }

    public Map<String, Object> importFile() throws Exception {
        Map<String, Object> results = new HashMap<String, Object>();
        
        bootstrap(file);
        agents.clear();
        missingTerms.clear();
        asael = 0;

        // Loading ASAE list
        int i = 1;
        while (i <= rowCount) {

            HSSFRow row = sh.getRow(i);
            if (row != null) {
                String nsc = getCellData("", i, row.getCell((short)headers.get("NSC")));
                String ctcae_category= getCellData("", i, row.getCell((short)headers.get("CTCAE_CATEGORY")));
                int ctcae_version = Integer.parseInt(getCellData("", i, row.getCell((short)headers.get("CTCAE_VERSION"))));
                String ae_term = getCellData("", i, row.getCell((short)headers.get("AE_TERM")));

                // System.out.println(String.format("NSC:%s,   V:%s,   C:%s,   T:%s", nsc, ctcae_version, ctcae_category, ae_term));

                Agent a = agents.get(nsc);
                if (a == null) {
                    a = agentDao.getByNscNumber(nsc);
                    // System.out.println(asael + ". OK. Found agent [" + a.getName() + "] for NSC: [" + nsc + "]");
                    agents.put(nsc, a);
                }
                
                if (a != null) {
                    AgentSpecificCtcTerm t = new AgentSpecificCtcTerm();
                    t.setAgent(a);

                    List<CtcTerm> list = terminologyRepository.getCtcTerm(ctcae_category, ctcae_version, ae_term);
                    if (list.size() == 0) {
                        // System.out.println("Err. Term not found: " + ae_term);
                        missingTerms.add(ae_term);
                    } else {
                        t.setCtcTerm(list.get(0));
                        if (persistASAE(t)) asael++;
                    }
                    
                } else {
                    // System.out.println("Err. The agent was not found by its NSC: " + nsc);
                }

            }
            i++;
        }

        results.put(KEY_MISSING_TERMS, missingTerms);
        results.put(KEY_PROCESSED_AGENTS, agents.size());
        results.put(KEY_PROCESSED_AGENTTERMS, asael);

/*
        System.out.println();
        System.out.println(String.format("Loaded %d agents", agents.size()));
        System.out.println(String.format("Built %d ASAE terms", asael));
        System.out.println("OK.");
*/

        return results;
    }

    private boolean isAgentSpecificTermPersisted(AgentSpecificTerm at) {
        List<AgentSpecificCtcTerm> l = agentSpecificTermDao.getCtcTerm(at.getAgent().getId(), at.getTerm().getId());
        if (l.size() > 0) return true;
        return false;
    }

    /*
    * Persisting ASAE list
    * */
    private boolean persistASAE(AgentSpecificTerm t) {
        if (!isAgentSpecificTermPersisted(t)) {
            agentSpecificTermDao.save(t);
            List<StudyAgent> l = getStudyAgentDao().getByAgentID(t.getAgent().getId());
            for (StudyAgent s : l) {
                asaelService.synchronizeStudyWithAgentTerm(s.getStudy(), t);
                studyDao.save(s.getStudy());
            }
            return true;
        }
        return false;
    }

    private void bootstrap(File inputFile) throws Exception {
        try {
            poifs = new POIFSFileSystem(new FileInputStream(inputFile));
        } catch (IOException e) {
            // to handle java.io.IOException: Invalid header signature; read -2300849302551019537, expected -2226271756974174256
            // if somebody knows how to get rid of the exception, please remove the try/catch.
        }

        // create a workbook out of the input stream
        wb = new HSSFWorkbook(poifs);
        sh = wb.getSheetAt(0);
        rowCount = sh.getLastRowNum();
        columnsCount = sh.getRow(0).getLastCellNum();

        // get needed headers
        for (byte i = 0; i<columnsCount; i++) {
            HSSFCell cell = sh.getRow(0).getCell(i);
            if (headers.containsKey(cell.getStringCellValue())) {
{'fraction_non_alphanumeric': 0.11874805840323081, 'fraction_numerical': 0.005591798695246971, 'mean_word_length': 2.388684210526316, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3321911', 'n_tokens_mistral': 4173, 'n_tokens_neox': 3998, 'n_words': 796}