<commit_before>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package org.mskcc.cbio.portal.scripts;

import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.Element;
import org.dom4j.Node;
import org.dom4j.io.SAXReader;
import org.mskcc.cbio.portal.dao.DaoCancerStudy;
import org.mskcc.cbio.portal.dao.DaoClinicalAttribute;
import org.mskcc.cbio.portal.dao.DaoClinicalData;
import org.mskcc.cbio.portal.dao.DaoClinicalEvent;
import org.mskcc.cbio.portal.dao.DaoException;
import org.mskcc.cbio.portal.dao.MySQLbulkLoader;
import org.mskcc.cbio.portal.model.CancerStudy;
import org.mskcc.cbio.portal.model.ClinicalAttribute;
import org.mskcc.cbio.portal.model.ClinicalData;
import org.mskcc.cbio.portal.model.ClinicalEvent;

/**
 *
 * @author jgao
 */
public final class ImportCaisesClinicalXML {
    
    private ImportCaisesClinicalXML() {}
    
    public static void main(String[] args) throws Exception {
//        args = new String[] {"--data","/Users/gaoj/projects/cbio-portal-data/studies/prad/su2c/data_clinical_caises.xml",
//            "--meta","/Users/gaoj/projects/cbio-portal-data/studies/prad/su2c/meta_clinical_caises.txt",
//            "--loadMode", "bulkLoad"};
        if (args.length < 4) {
            System.out.println("command line usage:  importCaisesXml --data <data_clinical_caises.xml> --meta <meta_clinical_caises.txt>");
            return;
        }
        
       OptionParser parser = new OptionParser();
       OptionSpec<String> data = parser.accepts( "data",
               "caises data file" ).withRequiredArg().describedAs( "data_clinical_caises.xml" ).ofType( String.class );
       OptionSpec<String> meta = parser.accepts( "meta",
               "meta (description) file" ).withRequiredArg().describedAs( "meta_clinical_caises.txt" ).ofType( String.class );
       parser.acceptsAll(Arrays.asList("dbmsAction", "loadMode"));
       OptionSet options = null;
      try {
         options = parser.parse( args );
         //exitJVM = !options.has(returnFromMain);
      } catch (OptionException e) {
          e.printStackTrace();
      }
       
       String dataFile = null;
       if( options.has( data ) ){
          dataFile = options.valueOf( data );
       }else{
           throw new Exception( "'data' argument required.");
       }

       String descriptorFile = null;
       if( options.has( meta ) ){
          descriptorFile = options.valueOf( meta );
       }else{
           throw new Exception( "'meta' argument required.");
       }
        
        Properties properties = new Properties();
        properties.load(new FileInputStream(descriptorFile));
      
        CancerStudy cancerStudy = DaoCancerStudy.getCancerStudyByStableId(properties.getProperty("cancer_study_identifier"));
        if (cancerStudy == null) {
            throw new Exception("Unknown cancer study: " + properties.getProperty("cancer_study_identifier"));
        }
        
        int cancerStudyId = cancerStudy.getInternalId();
        DaoClinicalEvent.deleteByCancerStudyId(cancerStudyId);
        
        importData(dataFile, cancerStudy.getInternalId());

        System.out.println("Done!");
    }
    
//    private static Map<String,String> readSampleIDMapping(String clinicalDataFile) throws IOException {
//        FileReader reader =  new FileReader(clinicalDataFile);
//        BufferedReader buff = new BufferedReader(reader);
//
//        Map<String,String> map = new HashMap<String,String>();
//        
//        String line = buff.readLine(); 
//        while (line.startsWith("#")) {
//            line = buff.readLine();
//        }
//        
//        Map<String,Integer> mapHeaderIndex = new HashMap<String,Integer>();
//        String[] headers = line.split("\t");
//        for (int i=0; i<headers.length; i++) {
//            String header = headers[i];
//            mapHeaderIndex.put(header, i);
//        }
//        
//        int ixSampleId = mapHeaderIndex.get("CASE_ID");
//        int ixSu2cSampleId = mapHeaderIndex.get("SU2C_SAMPLE_ID");
//        
//        while ((line = buff.readLine()) != null) {
//            String[] parts = line.split("\t");
//            if (!parts[1].isEmpty()) {
//                map.put(parts[ixSampleId], parts[ixSu2cSampleId]);
//            }
//        }
//        return map;
//    }
    
    private static void importData(String urlXml, int cancerStudyId) throws DocumentException, DaoException {
        MySQLbulkLoader.bulkLoadOn();
        
        SAXReader reader = new SAXReader();
        Document document = reader.read(urlXml);
        
        List<Node> patientNodes = document.selectNodes("//Patients/Patient");
        
        long clinicalEventId = DaoClinicalEvent.getLargestClinicalEventId();
        
        Map<String, Set<String>> mapPatientIdSampleId = getMapPatientIdSampleId(cancerStudyId);
        Map<String, Set<String>> mapSu2cSampleIdSampleId = getMapSu2cSampleIdSampleId(cancerStudyId);
        
        for (Node patientNode : patientNodes) {
            String patientId = patientNode.selectSingleNode("PtProtocolStudyId").getText();
            
            System.out.println("Importing "+patientId);

            // processing clinical data
            List<ClinicalData> clinicalData = filterClinicalData(
                    parsePatientClinicalData(patientNode, patientId, cancerStudyId),
                    mapPatientIdSampleId);
            clinicalData.addAll(filterClinicalData(
                    parseClinicalDataFromSpecimen(patientNode, cancerStudyId),
                    mapSu2cSampleIdSampleId));
            for (ClinicalData cd : clinicalData) {
                if (DaoClinicalData.getDatum(cancerStudyId, cd.getCaseId(), cd.getAttrId())==null) {
                    DaoClinicalData.addDatum(cd);
                }
            }
            // add unknow attriutes -- this 
            for (ClinicalAttribute ca : getClinicalAttributes()) {
                if (DaoClinicalAttribute.getDatum(ca.getAttrId())==null) {<commit_msg>check if clinical data has been imported when imporint caises data<commit_after>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package org.mskcc.cbio.portal.scripts;

import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.Element;
import org.dom4j.Node;
import org.dom4j.io.SAXReader;
import org.mskcc.cbio.portal.dao.DaoCancerStudy;
import org.mskcc.cbio.portal.dao.DaoClinicalAttribute;
import org.mskcc.cbio.portal.dao.DaoClinicalData;
import org.mskcc.cbio.portal.dao.DaoClinicalEvent;
import org.mskcc.cbio.portal.dao.DaoException;
import org.mskcc.cbio.portal.dao.MySQLbulkLoader;
import org.mskcc.cbio.portal.model.CancerStudy;
import org.mskcc.cbio.portal.model.ClinicalAttribute;
import org.mskcc.cbio.portal.model.ClinicalData;
import org.mskcc.cbio.portal.model.ClinicalEvent;

/**
 *
 * @author jgao
 */
public final class ImportCaisesClinicalXML {
    
    private ImportCaisesClinicalXML() {}
    
    public static void main(String[] args) throws Exception {
//        args = new String[] {"--data","/Users/gaoj/projects/cbio-portal-data/studies/prad/su2c/data_clinical_caises.xml",
//            "--meta","/Users/gaoj/projects/cbio-portal-data/studies/prad/su2c/meta_clinical_caises.txt",
//            "--loadMode", "bulkLoad"};
        if (args.length < 4) {
            System.out.println("command line usage:  importCaisesXml --data <data_clinical_caises.xml> --meta <meta_clinical_caises.txt>");
            return;
        }
        
       OptionParser parser = new OptionParser();
       OptionSpec<String> data = parser.accepts( "data",
               "caises data file" ).withRequiredArg().describedAs( "data_clinical_caises.xml" ).ofType( String.class );
       OptionSpec<String> meta = parser.accepts( "meta",
               "meta (description) file" ).withRequiredArg().describedAs( "meta_clinical_caises.txt" ).ofType( String.class );
       parser.acceptsAll(Arrays.asList("dbmsAction", "loadMode"));
       OptionSet options = null;
      try {
         options = parser.parse( args );
         //exitJVM = !options.has(returnFromMain);
      } catch (OptionException e) {
          e.printStackTrace();
      }
       
       String dataFile = null;
       if( options.has( data ) ){
          dataFile = options.valueOf( data );
       }else{
           throw new Exception( "'data' argument required.");
       }

       String descriptorFile = null;
       if( options.has( meta ) ){
          descriptorFile = options.valueOf( meta );
       }else{
           throw new Exception( "'meta' argument required.");
       }
        
        Properties properties = new Properties();
        properties.load(new FileInputStream(descriptorFile));
      
        CancerStudy cancerStudy = DaoCancerStudy.getCancerStudyByStableId(properties.getProperty("cancer_study_identifier"));
        if (cancerStudy == null) {
            throw new Exception("Unknown cancer study: " + properties.getProperty("cancer_study_identifier"));
        }
        
        int cancerStudyId = cancerStudy.getInternalId();
        DaoClinicalEvent.deleteByCancerStudyId(cancerStudyId);
        
        importData(dataFile, cancerStudy.getInternalId());

        System.out.println("Done!");
    }
    
//    private static Map<String,String> readSampleIDMapping(String clinicalDataFile) throws IOException {
//        FileReader reader =  new FileReader(clinicalDataFile);
//        BufferedReader buff = new BufferedReader(reader);
//
//        Map<String,String> map = new HashMap<String,String>();
//        
//        String line = buff.readLine(); 
//        while (line.startsWith("#")) {
//            line = buff.readLine();
//        }
//        
//        Map<String,Integer> mapHeaderIndex = new HashMap<String,Integer>();
//        String[] headers = line.split("\t");
//        for (int i=0; i<headers.length; i++) {
//            String header = headers[i];
//            mapHeaderIndex.put(header, i);
//        }
//        
//        int ixSampleId = mapHeaderIndex.get("CASE_ID");
//        int ixSu2cSampleId = mapHeaderIndex.get("SU2C_SAMPLE_ID");
//        
//        while ((line = buff.readLine()) != null) {
//            String[] parts = line.split("\t");
//            if (!parts[1].isEmpty()) {
//                map.put(parts[ixSampleId], parts[ixSu2cSampleId]);
//            }
//        }
//        return map;
//    }
    
    private static void importData(String urlXml, int cancerStudyId) throws Exception {
        MySQLbulkLoader.bulkLoadOn();
        
        SAXReader reader = new SAXReader();
        Document document = reader.read(urlXml);
        
        List<Node> patientNodes = document.selectNodes("//Patients/Patient");
        
        long clinicalEventId = DaoClinicalEvent.getLargestClinicalEventId();
        
        Map<String, Set<String>> mapPatientIdSampleId = getMapPatientIdSampleId(cancerStudyId);
        Map<String, Set<String>> mapSu2cSampleIdSampleId = getMapSu2cSampleIdSampleId(cancerStudyId);
        
        if (mapPatientIdSampleId.isEmpty()) {
            throw new Exception("clinical data need to be imported first");
        }
        
        for (Node patientNode : patientNodes) {
            String patientId = patientNode.selectSingleNode("PtProtocolStudyId").getText();
            
            System.out.println("Importing "+patientId);

            // processing clinical data
            List<ClinicalData> clinicalData = filterClinicalData(
                    parsePatientClinicalData(patientNode, patientId, cancerStudyId),
                    mapPatientIdSampleId);
            clinicalData.addAll(filterClinicalData(
                    parseClinicalDataFromSpecimen(patientNode, cancerStudyId),
                    mapSu2cSampleIdSampleId));
            for (ClinicalData cd : clinicalData) {
                if (DaoClinicalData.getDatum(cancerStudyId, cd.getCaseId(), cd.getAttrId())==null) {
                    DaoClinicalData.addDatum(cd);
                }
            }
            // add unknow attriutes -- this 
            for (ClinicalAttribute ca : getClinicalAttributes()) {
                if (DaoClinicalAttribute.getDatum(ca.getAttrId())==null) {
{'fraction_non_alphanumeric': 0.11980830670926518, 'fraction_numerical': 0.00243420051726761, 'mean_word_length': 2.5813129937346773, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5109781', 'n_tokens_mistral': 3922, 'n_tokens_neox': 3756, 'n_words': 710}