<commit_before>#!/usr/bin/perl
# file: moveGDACOverrides.pl
# author: <NAME>, <EMAIL>

use strict;
use warnings;
use File::Spec;
use File::Util;
use Data::Dumper;
use Getopt::Long;

use FirehoseTransformationWorkflow;
        # TODO: ACCESS CONTROL: change to CANCER_STUDY

my $usage = <<EOT;
usage:
moveGDACOverrides.pl

Given a set of override files in a directory, move them into a Firehose data set.

--DeepFirehoseDirectory <Firehose Directory>        # required; directory which stores firehose data
--customFileType                                    # indicator of the custom file type to move (Agilent MRNA, RNA Seq, CNA, MAF)
--customDirectory                                   # directory containing overrides
--RunDate                                           # rundate for the Firehose run
--customFilesToMoveFile <file containing pairs of custom_file cancer_type>
                                                    # required; 
EOT

# args:
# on laptop, for testing:
# --DeepFirehoseDirectory /Users/goldbera/Data/firehose/data/copyOfCurrent/data
# --customDirectory /Users/goldbera/Documents/workspace/cgds/data/MAFs --RunDate 20110327 
# --customFilesToMoveFile /Users/goldbera/Data/firehose/data/copyOfCurrent/specialMAFs.txt
# on buri:
# --DeepFirehoseDirectory /scratch/data/goldberg/firehoseData/tcga-data.nci.nih.gov/tcgafiles/ftp_auth/distro_ftpusers/tcga4yeo/other/gdacs/gdacbroad --customDirectory /home/goldberg/workspace/sander/cgds/data/MAFs --RunDate 20110421 --customFilesToMoveFile /home/goldberg/workspace/sander/import_and_convert_Firehose_data/config/specialMAFs.txt

my( $customDirectory, $customFileType, $DeepFirehoseDirectory, $runDate, $customFilesToMoveFile );

# make sure to put the customFile file in the proper gdac directory, 
# because that's what convertFirehoseData.pl will use via getLastestVersionOfFile()

# map - key is customFileType, value is gdac dir - file pair
my $customFileProperties = {
	'AGILENT-MRNA' => [ 'gdac.broadinstitute.org_<CANCER>.Merge_transcriptome__agilentg4502a_07_3__unc_edu__Level_3__unc_lowess_normalization_gene_level__data.Level_3.<date><version>', '<CANCER>.transcriptome__agilentg4502a_07_3__unc_edu__Level_3__unc_lowess_normalization_gene_level__data.data.txt'],
	'RNA-SEQ' => [ 'gdac.broadinstitute.org_<CANCER>.Merge_rnaseq__illumina<RNA-SEQ-PLATFORM>_rnaseq__unc_edu__Level_3__gene_expression__data.Level_3.<date><version>', '<CANCER>.rnaseq__illumina<RNA-SEQ-PLATFORM>_rnaseq__unc_edu__Level_3__gene_expression__data.data.txt'],
	'CNA' => [ 'gdac.broadinstitute.org_<CANCER>.CopyNumber_Gistic2.Level_4.<date><version>', 'all_thresholded.by_genes.txt'],
	'MAF' => [ 'gdac.broadinstitute.org_<CANCER>.Mutation_Assessor.Level_4.<date><version>', '<CANCER>.maf.annotated'],
};

main();
sub main{
	
    # process arg list
    GetOptions (
        "customDirectory=s" => \$customDirectory,
		"customFileType=s" => \$customFileType,
        "DeepFirehoseDirectory=s" => \$DeepFirehoseDirectory,
        "runDate=s" => \$runDate,
        "customFilesToMoveFile=s" => \$customFilesToMoveFile );

	unless(exists($customFileProperties->{$customFileType})) {
		warn "customFileType: $customFileType is not recognized.";
		exit;
	}
        
    my %customFilesToMove;
    my $f = File::Util->new();
    my @tmp = $f->load_file( $customFilesToMoveFile, '--as-lines' );
    foreach (@tmp){
    	my( $file, $cancer ) = split( /\s+/, $_ );
    	$customFilesToMove{$file} = $cancer;
    }

	my $destDir = $customFileProperties->{$customFileType}->[0];
	my $destFile = $customFileProperties->{$customFileType}->[1];

	foreach my $customFile (keys %customFilesToMove){
	    moveGDACOverridesFile( $customFile, $customFilesToMove{$customFile}, $destDir, $destFile );
	}
}

sub moveGDACOverridesFile{
	my( $customFile, $cancer, $destDir, $destFile ) = @_;
	# in override dir, cancer directory is called <CANCER>_tcga
	my @directories = ( $customDirectory, $cancer . '_tcga' );
	my $fromFile = File::Spec->catdir( @directories, $customFile );

    my $CancersFirehoseDataDir = File::Spec->catfile( $DeepFirehoseDirectory, $cancer, $runDate . '00' );

	# if we are using custom CNA, we will need to move over Log2CNA,<commit_msg>Minor optimization<commit_after>#!/usr/bin/perl
# file: moveGDACOverrides.pl
# author: <NAME>, <EMAIL>

use strict;
use warnings;
use File::Spec;
use File::Util;
use Data::Dumper;
use Getopt::Long;

use FirehoseTransformationWorkflow;
        # TODO: ACCESS CONTROL: change to CANCER_STUDY

my $usage = <<EOT;
usage:
moveGDACOverrides.pl

Given a set of override files in a directory, move them into a Firehose data set.

--DeepFirehoseDirectory <Firehose Directory>        # required; directory which stores firehose data
--customFileType                                    # indicator of the custom file type to move (Agilent MRNA, RNA Seq, CNA, MAF)
--customDirectory                                   # directory containing overrides
--RunDate                                           # rundate for the Firehose run
--customFilesToMoveFile <file containing pairs of custom_file cancer_type>
                                                    # required; 
EOT

# args:
# on laptop, for testing:
# --DeepFirehoseDirectory /Users/goldbera/Data/firehose/data/copyOfCurrent/data
# --customDirectory /Users/goldbera/Documents/workspace/cgds/data/MAFs --RunDate 20110327 
# --customFilesToMoveFile /Users/goldbera/Data/firehose/data/copyOfCurrent/specialMAFs.txt
# on buri:
# --DeepFirehoseDirectory /scratch/data/goldberg/firehoseData/tcga-data.nci.nih.gov/tcgafiles/ftp_auth/distro_ftpusers/tcga4yeo/other/gdacs/gdacbroad --customDirectory /home/goldberg/workspace/sander/cgds/data/MAFs --RunDate 20110421 --customFilesToMoveFile /home/goldberg/workspace/sander/import_and_convert_Firehose_data/config/specialMAFs.txt

my( $customDirectory, $customFileType, $DeepFirehoseDirectory, $runDate, $customFilesToMoveFile );

# make sure to put the customFile file in the proper gdac directory, 
# because that's what convertFirehoseData.pl will use via getLastestVersionOfFile()

# map - key is customFileType, value is gdac dir - file pair
my $customFileProperties = {
	'AGILENT-MRNA' => [ 'gdac.broadinstitute.org_<CANCER>.Merge_transcriptome__agilentg4502a_07_3__unc_edu__Level_3__unc_lowess_normalization_gene_level__data.Level_3.<date><version>', '<CANCER>.transcriptome__agilentg4502a_07_3__unc_edu__Level_3__unc_lowess_normalization_gene_level__data.data.txt'],
	'RNA-SEQ' => [ 'gdac.broadinstitute.org_<CANCER>.Merge_rnaseq__illumina<RNA-SEQ-PLATFORM>_rnaseq__unc_edu__Level_3__gene_expression__data.Level_3.<date><version>', '<CANCER>.rnaseq__illumina<RNA-SEQ-PLATFORM>_rnaseq__unc_edu__Level_3__gene_expression__data.data.txt'],
	'CNA' => [ 'gdac.broadinstitute.org_<CANCER>.CopyNumber_Gistic2.Level_4.<date><version>', 'all_thresholded.by_genes.txt'],
	'MAF' => [ 'gdac.broadinstitute.org_<CANCER>.Mutation_Assessor.Level_4.<date><version>', '<CANCER>.maf.annotated'],
};

main();
sub main{
	
    # process arg list
    GetOptions (
        "customDirectory=s" => \$customDirectory,
		"customFileType=s" => \$customFileType,
        "DeepFirehoseDirectory=s" => \$DeepFirehoseDirectory,
        "runDate=s" => \$runDate,
        "customFilesToMoveFile=s" => \$customFilesToMoveFile );

	unless(exists($customFileProperties->{$customFileType})) {
		warn "customFileType: $customFileType is not recognized.";
		exit;
	}
        
    my %customFilesToMove;
    my $f = File::Util->new();
    my @tmp = $f->load_file( $customFilesToMoveFile, '--as-lines' );
    foreach (@tmp){
    	my( $file, $cancer ) = split( /\s+/, $_ );
    	$customFilesToMove{$file} = $cancer;
    }

	my $destDir = $customFileProperties->{$customFileType}->[0];
	my $destFile = $customFileProperties->{$customFileType}->[1];

	foreach my $customFile (keys %customFilesToMove){
	    moveGDACOverridesFile( $customFile, $customFilesToMove{$customFile}, $destDir, $destFile );
	}
}

sub moveGDACOverridesFile{
	my( $customFile, $cancer, $destDir, $destFile ) = @_;
	# in override dir, cancer directory is called <CANCER>_tcga
	my $fromFile = File::Spec->catdir( ($customDirectory, $cancer . '_tcga' ), $customFile );

    my $CancersFirehoseDataDir = File::Spec->catfile( $DeepFirehoseDirectory, $cancer, $runDate . '00' );

	# if we are using custom CNA, we will need to move over Log2CNA,
{'fraction_non_alphanumeric': 0.124446571736269, 'fraction_numerical': 0.010769414861792509, 'mean_word_length': 4.283185840707965, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6945066', 'n_tokens_mistral': 2939, 'n_tokens_neox': 2782, 'n_words': 533}