<commit_before>#!/usr/bin/perl -w
use strict;
use CoGeX;
use CoGe::Accessory::Web qw(get_defaults);
use CoGe::Accessory::Utils qw( trim execute );
use CoGe::Core::Notebook qw(add_items_to_notebook);
use File::Path;
use File::Basename;
use File::Spec::Functions qw(catdir catfile);
use File::Touch;
use File::Copy qw(copy);
use Getopt::Long;
use URI::Escape::JavaScript qw(escape unescape);
use Data::Dumper;

use vars qw(
    $staging_dir $file_str $notebook_name $notebook_desc $gid $nid $user_name
    $config $log_file $user $genome $notebook $result_dir $wid @failed_experiments
);

my $DELIMITER = '\t';

#-------------------------------------------------------------------------------

GetOptions(
    "staging_dir=s" => \$staging_dir,    # temporary staging path
    "result_dir=s"  => \$result_dir,     # results path
    "files=s"       => \$file_str,       # input data file (JS escape)
    "name=s"        => \$notebook_name,  # notebook name (JS escaped) if not nid
    "desc=s"        => \$notebook_desc,  # notebook description (JS escaped)
    "gid=s"         => \$gid,            # genome id
    "nid=s"         => \$nid,            # optional notebook id, otherwise new notebook created
    "wid=s"         => \$wid,            # workflow id
    "user_name=s"   => \$user_name,      # user name
    "config=s"      => \$config,         # CoGe config file
);

$| = 1;
die unless ($staging_dir);
mkpath($staging_dir) unless -r $staging_dir;
print "Starting $0 (pid $$)\n", qx/ps -o args $$/;

# Process and verify parameters
$file_str      = unescape($file_str);
$notebook_name = unescape($notebook_name);
$notebook_desc = unescape($notebook_desc);

if ($user_name eq 'public') {
    print "log: error: not logged in\n";
    exit(-1);
}

unless ($gid) {
    print "log: error: genome id parameter 'gid' not specified\n";
    exit(-1);
}

# Load config file
my $P;
if ($config) {
    $P = CoGe::Accessory::Web::get_defaults($config);
}
else {
    $P = CoGe::Accessory::Web::get_defaults();
}

# Connect to database
my $connstr = "dbi:mysql:dbname=".$P->{DBNAME}.";host=".$P->{DBHOST}.";port=".$P->{DBPORT}.";";
my $coge = CoGeX->connect( $connstr, $P->{DBUSER}, $P->{DBPASS} );
unless ($coge) {
    print "log: error: couldn't connect to database\n";
    exit(-1);
}

# Retrieve genome
if ($gid) {
    $genome = $coge->resultset('Genome')->find( { genome_id => $gid } );
    unless ($genome) {
        print "log: error finding genome id$gid\n";
        exit(-1);
    }
}

# Retrieve notebook
if ($nid) {
    $notebook = $coge->resultset('List')->find( { list_id => $nid } );
    unless ($notebook) {
        print "log: error finding notebook id$nid\n";
        exit(-1);
    }
}

# Retrieve user
$user = $coge->resultset('User')->find( { user_name => $user_name } );
unless ($user) {
    print "log: error finding user '$user_name'\n";
    exit(-1);
}

# Process each file into staging area
my $data_dir = catdir($staging_dir, 'data');
mkpath($data_dir);
my @files = split( ',', $file_str );
foreach my $file (@files) {
    my $filename = basename($file);

    # Decompress file (if necessary) into staging area
    if ( $file =~ /\.gz$/ ) {
        print "log: Decompressing '$filename'\n";
        $file =~ s/\.gz$//;
        run( $P->{GUNZIP} . ' -c ' . $file.'.gz' . ' > ' . $file );
    }

    # Untar file (if necessary) into staging area - TODO: do this before gunzip if tar.gz file
    if ( $file =~ /\.tar$/ ) {
        print "log: Extracting files\n";
        run( $P->{TAR}.' -xf '.$file.' --directory '.$data_dir );
    }
    else {
        run("cp $file $data_dir/$filename");
#        unless ( copy( $file, catfile($data_dir, $filename) ) ) {
#            print "log: error copying file:\n";
#            print "log: $!\n";
#            exit(-1);
#       }
    }
}

# Find metadata file
my ($metadata_file) = glob("$data_dir/*.txt");
unless ($metadata_file) { # require a metadata file for now but could be optional in the future
    print "log: error: cannot find .txt metadata file\n";
    exit(-1);
}

# Load metadata file
my $metadata;
print "log: Loading metadata file '".fileparse($metadata_file)."'\n";<commit_msg>Remove decompression step, issue 501<commit_after>#!/usr/bin/perl -w
use strict;
use CoGeX;
use CoGe::Accessory::Web qw(get_defaults);
use CoGe::Accessory::Utils qw( trim execute );
use CoGe::Core::Notebook qw(add_items_to_notebook);
use File::Path;
use File::Basename;
use File::Spec::Functions qw(catdir catfile);
use File::Touch;
use File::Copy qw(copy);
use Getopt::Long;
use URI::Escape::JavaScript qw(escape unescape);
use Data::Dumper;

use vars qw(
    $staging_dir $file_str $notebook_name $notebook_desc $gid $nid $user_name
    $config $log_file $user $genome $notebook $result_dir $wid @failed_experiments
);

my $DELIMITER = '\t';

#-------------------------------------------------------------------------------

GetOptions(
    "staging_dir=s" => \$staging_dir,    # temporary staging path
    "result_dir=s"  => \$result_dir,     # results path
    "files=s"       => \$file_str,       # input data file (JS escape)
    "name=s"        => \$notebook_name,  # notebook name (JS escaped) if not nid
    "desc=s"        => \$notebook_desc,  # notebook description (JS escaped)
    "gid=s"         => \$gid,            # genome id
    "nid=s"         => \$nid,            # optional notebook id, otherwise new notebook created
    "wid=s"         => \$wid,            # workflow id
    "user_name=s"   => \$user_name,      # user name
    "config=s"      => \$config,         # CoGe config file
);

$| = 1;
die unless ($staging_dir);
mkpath($staging_dir) unless -r $staging_dir;
print "Starting $0 (pid $$)\n", qx/ps -o args $$/;

# Process and verify parameters
$file_str      = unescape($file_str);
$notebook_name = unescape($notebook_name);
$notebook_desc = unescape($notebook_desc);

if ($user_name eq 'public') {
    print "log: error: not logged in\n";
    exit(-1);
}

unless ($gid) {
    print "log: error: genome id parameter 'gid' not specified\n";
    exit(-1);
}

# Load config file
my $P;
if ($config) {
    $P = CoGe::Accessory::Web::get_defaults($config);
}
else {
    $P = CoGe::Accessory::Web::get_defaults();
}

# Connect to database
my $connstr = "dbi:mysql:dbname=".$P->{DBNAME}.";host=".$P->{DBHOST}.";port=".$P->{DBPORT}.";";
my $coge = CoGeX->connect( $connstr, $P->{DBUSER}, $P->{DBPASS} );
unless ($coge) {
    print "log: error: couldn't connect to database\n";
    exit(-1);
}

# Retrieve genome
if ($gid) {
    $genome = $coge->resultset('Genome')->find( { genome_id => $gid } );
    unless ($genome) {
        print "log: error finding genome id$gid\n";
        exit(-1);
    }
}

# Retrieve notebook
if ($nid) {
    $notebook = $coge->resultset('List')->find( { list_id => $nid } );
    unless ($notebook) {
        print "log: error finding notebook id$nid\n";
        exit(-1);
    }
}

# Retrieve user
$user = $coge->resultset('User')->find( { user_name => $user_name } );
unless ($user) {
    print "log: error finding user '$user_name'\n";
    exit(-1);
}

# Process each file into staging area
my $data_dir = catdir($staging_dir, 'data');
mkpath($data_dir);
my @files = split( ',', $file_str );
foreach my $file (@files) {
    my $filename = basename($file);

    # Decompress file (if necessary) - removed, let the load_genome/experiment do this
#    if ( $file =~ /\.gz$/ ) {
#        print "log: Decompressing '$filename'\n";
#        $filename =~ s/\.gz$//;
#        run( $P->{GUNZIP}.' -c '.$file.' > '.catfile($data_dir, $filename) );
#        next;
#    }

    # Untar file (if necessary) - TODO: do this before gunzip if tar.gz file
    if ( $file =~ /\.tar$/ || $file =~ /\.tar\.gz$/) {
        print "log: Extracting files\n";
        run( $P->{TAR}.' -xf '.$file.' --directory '.$data_dir );
        next;
    }
    
    # Otherwise just copy the file as is
    run("cp $file $data_dir/$filename");
#    unless ( copy( $file, catfile($data_dir, $filename) ) ) {
#        print "log: error copying file:\n";
#        print "log: $!\n";
#        exit(-1);
#   }
}

# Find metadata file
my ($metadata_file) = glob("$data_dir/*.txt");
unless ($metadata_file) { # require a metadata file for now but could be optional in the future
    print "log: error: cannot find .txt metadata file\n";
    exit(-1);
}

# Load metadata file
my $metadata;
print "log: Loading metadata file '".fileparse($metadata_file)."'\n";
{'fraction_non_alphanumeric': 0.19522338403041825, 'fraction_numerical': 0.002732889733840304, 'mean_word_length': 2.8189655172413794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1033032', 'n_tokens_mistral': 3004, 'n_tokens_neox': 2899, 'n_words': 774}