<commit_before>package LoadCGDSdata;

require Exporter;
@ISA = qw(Exporter);
@EXPORT = qw( run set_up_classpath loadCGDScancerMutationDataFile ); 

use strict;
use warnings;
use Getopt::Long;
use File::Spec;
use File::Util;
use Data::Dumper;
use Env::Path;
use FirehoseEnv;

use Utilities;

# SUMMARY
# Load CGDS files into dbms (LoadCGDSdata.pm)
# Uses CGDS Java library to load CGDS input files into dbms
# Issues: 
# Testing: No unit testing; LoadCGDSdata.t does nothing

# Uses CGDS to load dbms from CGDS input files
# Step 2 in Firehose data processing pipeline
# convertFirehoseData.pl => loadCGDSdata.pl => load Tomcat => view in web portal
 
# structure of $CGDSinputData data:
# each cancer has its own directory; 
# data
# data/cancers
# data/cancers/gbm
# data/cancers/gbm/case_lists
# data/cancers/gbm/case_lists/cases_CGH.txt
# o o o  other case lists
# data/cancers/gbm/data_CNA.txt
# data/cancers/gbm/data_expression_median.txt
# data/cancers/gbm/gbm.txt
# data/cancers/gbm/meta_CNA.txt
# data/cancers/gbm/meta_miRNA.txt
# data/cancers/gbm/meta_expression_median.txt
# data/cancers/stad
# o o o  data for stad
# o o o  other cancers
# human gene list
# data/human_gene_info_2011_01_19

# o o o  important dirs in $cgdsHome
# build: compiled code, web stuff
# lib: jars
# bin: scripts
# web: web source
# testData: test data files

# globals
my $fileUtil;

# Given a set of well-structured CGDS input files, load them into the CGDS database. Works for files converted from Firehose.
sub run{
	
	my ($class, 
		$cgdsHome,            # directory of CGDS code
		$CGDSinputData,       # directory of CGDS input data
		$Cancers,            # full filename of cancers file
		$GeneFile,            # full filename of gene file
		$miRNAfile,           # full filename of miRNAs file
		$rppafile,           # full filename of rppa antibodies file
		$sangerfile,           # full filename of sanger census file
		$uniprotMappingFile,   # full filename of uniprot mapping file
        $nameOfPerCancerGermlineWhitelist,  # base filename of per cancer germline whitelist file, if any
        $nameOfPerCancerSomaticWhitelist,   # base filename of per cancer somatic whitelist file, if any
		$loadMutationArguments,   # global mutation loading arguments, passed without modification to org.mskcc.cgds.scripts.ImportProfileData
		                          # includes optional --acceptRemainingMutations and 
		                          # --somaticWhiteList <full filename of universal somatic whitelist file> if supplied
		) = @_;

	# check that required options are set
	my $util = Utilities->new( "" );
	$util->verifyArgumentsAreDefined( $cgdsHome, $CGDSinputData, $GeneFile, $miRNAfile, $rppafile, $sangerfile, $uniprotMappingFile );

	my $cmdLineCP = set_up_classpath( $cgdsHome );
	
	# Clear the Database
	# database parameters found in build.properties
	my $c = "$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ResetDatabase";
	print "Running: $c\n";
	system ($c);

	$fileUtil = File::Util->new();

    # Load up cancers
    system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportTypesOfCancers " . $Cancers ); 

	# TODO: control or surpress progress messages
	# Load up Entrez Genes
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportGeneData " . $GeneFile ); 
	
	# Load up all microRNA IDs
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportMicroRNAIDs " . $miRNAfile );  

	# Load up RPPA -Antibodies
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportProteinArrayInfo " . $rppafile );  

	# Load up Sanger Data
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportSangerCensusData " . $sangerfile );  

	# Load up UniProt ID Mapping (for Mutation Diagrams)
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportUniProtIdMapping " . $uniprotMappingFile );
	    
    load_cancer_data( $cgdsHome, $CGDSinputData, $cmdLineCP, $nameOfPerCancerGermlineWhitelist, 
        $nameOfPerCancerSomaticWhitelist, $loadMutationArguments );
    
}

# TODO: HIGH: IMPORTANT: UNIT TESTS
# todo: documentation

# todo: delete empty dirs: scripts/, notes/, cyto/, results/

# Set up Classpath to use all JAR files in lib dir
# should be O/S portable, but not tested
sub set_up_classpath{
	my $theCGDShome = shift;
	unless( defined( $theCGDShome ) ){
		warn "\$theCGDShome not defined ";
	}
	my $classpath = Env::Path->CLASSPATH;
	$classpath->Append( File::Spec->catfile( $theCGDShome, qw(  build WEB-INF classes ) ) );
	
	my @jar_files = glob( File::Spec->catfile( $theCGDShome, qw(  lib *.jar ) ) );
	foreach my $jar (@jar_files) {
	    $classpath->Append( $jar );
	}
	$classpath->Uniqify;
	my $cmdLineCP = join( Env::Path->PathSeparator, $classpath->List );
    # print "jars and dirs:\n", join( "\n", $classpath->List), "\n";
	return $cmdLineCP;
}

# load all cancer data
sub load_cancer_data{
  my( $cgdsHome, $theCGDSinputFiles, $cmdLineCP, $nameOfPerCancerGermlineWhitelist, 
	  $nameOfPerCancerSomaticWhitelist, $loadMutationArguments ) = @_;

  # for each cancer
  my @cancerDataDirs = $fileUtil->list_dir( File::Spec->catfile( $theCGDSinputFiles ), '--dirs-only', '--no-fsdots' );
  foreach my $cancerDataDir (@cancerDataDirs){
	    
	print "Loading $cancerDataDir.\n";
	
	# import all data
	# import cancer's name and metadata
	if( importCancerStudy( $cgdsHome, $theCGDSinputFiles, $cancerDataDir, $cmdLineCP ) ){

	  my @pathToDataFile = ( $theCGDSinputFiles, $cancerDataDir );
		  
	  # import case lists, e.g., ./importCaseList.pl $CGDS_HOME/data/ovarian/case_lists   
	  importCaseLists( $cgdsHome, $theCGDSinputFiles, $cancerDataDir, $cmdLineCP );
	        
	  # TODO: import clinical data
	  my $clinicalDataFile = $cancerDataDir . $Utilities::clinicalFileSuffix;
	  my $fullCanonicalClinicalDataFile = File::Spec->catfile( @pathToDataFile, $clinicalDataFile );
	  if ( $fileUtil->existent($fullCanonicalClinicalDataFile) ) {
		print "importingClinicalData: $fullCanonicalClinicalDataFile\n";
		system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportClinicalData " . $fullCanonicalClinicalDataFile ); 
	  }
	        
	  # import a cancer's data
	  importCancersData( $cgdsHome, $theCGDSinputFiles, File::Spec->catfile( $theCGDSinputFiles, $cancerDataDir ),
						 $cancerDataDir, $cmdLineCP, $nameOfPerCancerGermlineWhitelist,
						 $nameOfPerCancerSomaticWhitelist, $loadMutationArguments  );

	  # import rppa
	  my $fullCanonicalRPPADataFile = File::Spec->catfile( @pathToDataFile, 'data_rppa.txt' );
	  if ( $fileUtil->existent($fullCanonicalRPPADataFile) ) {
		print "importingRPPAData: $fullCanonicalRPPADataFile\n";
		system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportProteinArrayData " . $fullCanonicalRPPADataFile . ' ' . $cancerDataDir ); 
	  }
			
	  # import mutsig
	  my $fullCanonicalMutSigDataFile = File::Spec->catfile( @pathToDataFile, 'data_mutsig.txt' );
	  if ( $fileUtil->existent($fullCanonicalMutSigDataFile) ) {
		print "importingMutSigData: $fullCanonicalMutSigDataFile\n";
		#system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportMutSigData " . $fullCanonicalRPPADataFile . ' ' . $cancerDataDir ); 
	  }
		  
	  print "timestamp: ", timing(), "Loading $cancerDataDir complete.\n";
	}
  }
}   

# load a cancer's data; load each file prefixed by $Utilities::dataFilePrefix
sub importCancersData{
    my( $cgdsHome, $theCGDSinputFiles, $directory, $cancerDataDir, $cmdLineCP, $nameOfPerCancerGermlineWhitelist,
        $nameOfPerCancerSomaticWhitelist, $loadMutationArguments ) = @_;

    print "importCancersData: load data from '$directory'.\n";
    
    my @dataFiles = grep( /^$Utilities::dataFilePrefix/, $fileUtil->list_dir( $directory, '--files-only' ) );  
    print "importCancersData: data files: ", join( " ", @dataFiles ), "\n";
    foreach my $dataFile (@dataFiles){
        # handle mutation files separately
        if( $dataFile =~ /mutation/i ){
            loadCGDScancerMutationDataFile( $cgdsHome, $directory, File::Spec->catfile( $directory, $dataFile ), 
                $cmdLineCP, 
                $nameOfPerCancerGermlineWhitelist, $nameOfPerCancerSomaticWhitelist, $loadMutationArguments );
        }
		# handle rppa data separately
		elsif ( $dataFile =~ /rppa/i ) {
		  next;
		}
		# handle mutsig separately
		elsif ( $dataFile =~ /mutsig/i ) {
		  next;
		}<commit_msg>update to loadmutsig call<commit_after>package LoadCGDSdata;

require Exporter;
@ISA = qw(Exporter);
@EXPORT = qw( run set_up_classpath loadCGDScancerMutationDataFile ); 

use strict;
use warnings;
use Getopt::Long;
use File::Spec;
use File::Util;
use Data::Dumper;
use Env::Path;
use FirehoseEnv;

use Utilities;

# SUMMARY
# Load CGDS files into dbms (LoadCGDSdata.pm)
# Uses CGDS Java library to load CGDS input files into dbms
# Issues: 
# Testing: No unit testing; LoadCGDSdata.t does nothing

# Uses CGDS to load dbms from CGDS input files
# Step 2 in Firehose data processing pipeline
# convertFirehoseData.pl => loadCGDSdata.pl => load Tomcat => view in web portal
 
# structure of $CGDSinputData data:
# each cancer has its own directory; 
# data
# data/cancers
# data/cancers/gbm
# data/cancers/gbm/case_lists
# data/cancers/gbm/case_lists/cases_CGH.txt
# o o o  other case lists
# data/cancers/gbm/data_CNA.txt
# data/cancers/gbm/data_expression_median.txt
# data/cancers/gbm/gbm.txt
# data/cancers/gbm/meta_CNA.txt
# data/cancers/gbm/meta_miRNA.txt
# data/cancers/gbm/meta_expression_median.txt
# data/cancers/stad
# o o o  data for stad
# o o o  other cancers
# human gene list
# data/human_gene_info_2011_01_19

# o o o  important dirs in $cgdsHome
# build: compiled code, web stuff
# lib: jars
# bin: scripts
# web: web source
# testData: test data files

# globals
my $fileUtil;

# Given a set of well-structured CGDS input files, load them into the CGDS database. Works for files converted from Firehose.
sub run{
	
	my ($class, 
		$cgdsHome,            # directory of CGDS code
		$CGDSinputData,       # directory of CGDS input data
		$Cancers,            # full filename of cancers file
		$GeneFile,            # full filename of gene file
		$miRNAfile,           # full filename of miRNAs file
		$rppafile,           # full filename of rppa antibodies file
		$sangerfile,           # full filename of sanger census file
		$uniprotMappingFile,   # full filename of uniprot mapping file
        $nameOfPerCancerGermlineWhitelist,  # base filename of per cancer germline whitelist file, if any
        $nameOfPerCancerSomaticWhitelist,   # base filename of per cancer somatic whitelist file, if any
		$loadMutationArguments,   # global mutation loading arguments, passed without modification to org.mskcc.cgds.scripts.ImportProfileData
		                          # includes optional --acceptRemainingMutations and 
		                          # --somaticWhiteList <full filename of universal somatic whitelist file> if supplied
		) = @_;

	# check that required options are set
	my $util = Utilities->new( "" );
	$util->verifyArgumentsAreDefined( $cgdsHome, $CGDSinputData, $GeneFile, $miRNAfile, $rppafile, $sangerfile, $uniprotMappingFile );

	my $cmdLineCP = set_up_classpath( $cgdsHome );
	
	# Clear the Database
	# database parameters found in build.properties
	my $c = "$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ResetDatabase";
	print "Running: $c\n";
	system ($c);

	$fileUtil = File::Util->new();

    # Load up cancers
    system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportTypesOfCancers " . $Cancers ); 

	# TODO: control or surpress progress messages
	# Load up Entrez Genes
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportGeneData " . $GeneFile ); 
	
	# Load up all microRNA IDs
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportMicroRNAIDs " . $miRNAfile );  

	# Load up RPPA -Antibodies
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportProteinArrayInfo " . $rppafile );  

	# Load up Sanger Data
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportSangerCensusData " . $sangerfile );  

	# Load up UniProt ID Mapping (for Mutation Diagrams)
	system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportUniProtIdMapping " . $uniprotMappingFile );
	    
    load_cancer_data( $cgdsHome, $CGDSinputData, $cmdLineCP, $nameOfPerCancerGermlineWhitelist, 
        $nameOfPerCancerSomaticWhitelist, $loadMutationArguments );
    
}

# TODO: HIGH: IMPORTANT: UNIT TESTS
# todo: documentation

# todo: delete empty dirs: scripts/, notes/, cyto/, results/

# Set up Classpath to use all JAR files in lib dir
# should be O/S portable, but not tested
sub set_up_classpath{
	my $theCGDShome = shift;
	unless( defined( $theCGDShome ) ){
		warn "\$theCGDShome not defined ";
	}
	my $classpath = Env::Path->CLASSPATH;
	$classpath->Append( File::Spec->catfile( $theCGDShome, qw(  build WEB-INF classes ) ) );
	
	my @jar_files = glob( File::Spec->catfile( $theCGDShome, qw(  lib *.jar ) ) );
	foreach my $jar (@jar_files) {
	    $classpath->Append( $jar );
	}
	$classpath->Uniqify;
	my $cmdLineCP = join( Env::Path->PathSeparator, $classpath->List );
    # print "jars and dirs:\n", join( "\n", $classpath->List), "\n";
	return $cmdLineCP;
}

# load all cancer data
sub load_cancer_data{
  my( $cgdsHome, $theCGDSinputFiles, $cmdLineCP, $nameOfPerCancerGermlineWhitelist, 
	  $nameOfPerCancerSomaticWhitelist, $loadMutationArguments ) = @_;

  # for each cancer
  my @cancerDataDirs = $fileUtil->list_dir( File::Spec->catfile( $theCGDSinputFiles ), '--dirs-only', '--no-fsdots' );
  foreach my $cancerDataDir (@cancerDataDirs){
	    
	print "Loading $cancerDataDir.\n";
	
	# import all data
	# import cancer's name and metadata
	if( importCancerStudy( $cgdsHome, $theCGDSinputFiles, $cancerDataDir, $cmdLineCP ) ){

	  my @pathToDataFile = ( $theCGDSinputFiles, $cancerDataDir );
		  
	  # import case lists, e.g., ./importCaseList.pl $CGDS_HOME/data/ovarian/case_lists   
	  importCaseLists( $cgdsHome, $theCGDSinputFiles, $cancerDataDir, $cmdLineCP );
	        
	  # TODO: import clinical data
	  my $clinicalDataFile = $cancerDataDir . $Utilities::clinicalFileSuffix;
	  my $fullCanonicalClinicalDataFile = File::Spec->catfile( @pathToDataFile, $clinicalDataFile );
	  if ( $fileUtil->existent($fullCanonicalClinicalDataFile) ) {
		print "importingClinicalData: $fullCanonicalClinicalDataFile\n";
		system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportClinicalData " . $fullCanonicalClinicalDataFile ); 
	  }
	        
	  # import a cancer's data
	  importCancersData( $cgdsHome, $theCGDSinputFiles, File::Spec->catfile( $theCGDSinputFiles, $cancerDataDir ),
						 $cancerDataDir, $cmdLineCP, $nameOfPerCancerGermlineWhitelist,
						 $nameOfPerCancerSomaticWhitelist, $loadMutationArguments  );

	  # import rppa
	  my $fullCanonicalRPPADataFile = File::Spec->catfile( @pathToDataFile, 'data_rppa.txt' );
	  if ( $fileUtil->existent($fullCanonicalRPPADataFile) ) {
		print "importingRPPAData: $fullCanonicalRPPADataFile\n";
		system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportProteinArrayData " . $fullCanonicalRPPADataFile . ' ' . $cancerDataDir ); 
	  }
			
	  # import mutsig
	  my $fullCanonicalMutSigDataFile = File::Spec->catfile( @pathToDataFile, 'data_mutsig.txt' );
	  my $fullCanonicalMutSigMetaFile = File::Spec->catfile( @pathToDataFile, 'meta_mutsig.txt' );
	  if ( $fileUtil->existent($fullCanonicalMutSigDataFile) && $fileUtil->existent($fullCanonicalMutSigMetaFile)) {
		print "importingMutSigData: $fullCanonicalMutSigDataFile\n";
		system ("$JAVA_HOME/bin/java -Xmx1524M -cp $cmdLineCP -DCGDS_HOME='$cgdsHome' org.mskcc.cgds.scripts.ImportMutSigData " . $fullCanonicalMutSigDataFile . ' ' . $fullCanonicalMutSigMetaFile ); 
	  }
		  
	  print "timestamp: ", timing(), "Loading $cancerDataDir complete.\n";
	}
  }
}   

# load a cancer's data; load each file prefixed by $Utilities::dataFilePrefix
sub importCancersData{
    my( $cgdsHome, $theCGDSinputFiles, $directory, $cancerDataDir, $cmdLineCP, $nameOfPerCancerGermlineWhitelist,
        $nameOfPerCancerSomaticWhitelist, $loadMutationArguments ) = @_;

    print "importCancersData: load data from '$directory'.\n";
    
    my @dataFiles = grep( /^$Utilities::dataFilePrefix/, $fileUtil->list_dir( $directory, '--files-only' ) );  
    print "importCancersData: data files: ", join( " ", @dataFiles ), "\n";
    foreach my $dataFile (@dataFiles){
        # handle mutation files separately
        if( $dataFile =~ /mutation/i ){
            loadCGDScancerMutationDataFile( $cgdsHome, $directory, File::Spec->catfile( $directory, $dataFile ), 
                $cmdLineCP, 
                $nameOfPerCancerGermlineWhitelist, $nameOfPerCancerSomaticWhitelist, $loadMutationArguments );
        }
		# handle rppa data separately
		elsif ( $dataFile =~ /rppa/i ) {
		  next;
		}
		# handle mutsig separately
		elsif ( $dataFile =~ /mutsig/i ) {
		  next;
		}
{'fraction_non_alphanumeric': 0.11622918430708438, 'fraction_numerical': 0.005532034998588766, 'mean_word_length': 4.090804597701149, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1441413', 'n_tokens_mistral': 6580, 'n_tokens_neox': 6040, 'n_words': 1431}