<commit_before>#!/usr/bin/env perl
# Manage a Lyve-SET project
# Author: <NAME> <<EMAIL>>

use strict;
use warnings;
use Data::Dumper;
use Getopt::Long;
use File::Spec;
use Cwd qw/realpath/;
use File::Basename qw/basename/;
use File::Copy qw/copy move/;
use File::Slurp qw/read_file/;
use Bio::Perl;

use FindBin;
use lib "$FindBin::RealBin/../lib";
$ENV{PATH}="$ENV{PATH}:$FindBin::RealBin/../lib/edirect";

# The directories a project should have
my @projectSubdir=qw(vcf vcf/unfiltered msa bam reads reference tmp asm log);

# get project test data
my $testDir="$FindBin::RealBin/../testdata";
my @test=_uniq(map({basename($_) if(-d $_)} glob("$testDir/*")));
my $testdata=join(", ",@test);

sub logmsg{local $0=basename $0; print STDERR "$0: @_\n";}
exit main();

sub main{
  my $settings={};
  GetOptions($settings,qw(help create test=s delete add-reads=s remove-reads=s add-assembly=s remove-assembly=s change-reference=s)) or die $!;
  die usage() if($$settings{help});
  die "ERROR: need a SET project\n".usage() if(!@ARGV);
  my $project=shift(@ARGV);
  $$settings{create}=1 if($$settings{test});

  createProjectDir($project,$settings) if($$settings{create});
  if(!is_project($project,$settings)){
    my(undef,$dirs)=is_project($project,$settings);
    die "$project is not a SET project! Missing $dirs";
  }
  $$settings{tempdir}||="$project/tmp";

  addSampleData($project,$settings) if($$settings{test});
  addReads($project,$settings) if($$settings{'add-reads'});
  addAssembly($project,$settings) if($$settings{'add-assembly'});
  changeReference($project,$settings) if($$settings{'change-reference'});
  removeAssembly($project,$settings) if($$settings{'remove-assembly'});
  removeReads($project,$settings) if($$settings{'remove-reads'});
  deleteProject($project,$settings) if($$settings{delete});
  removeGenome($project,$settings) if($$settings{'remove-genome'});

  return 0;
}

sub createProjectDir{
  my($dir,$settings)=@_;
  die "ERROR: directory $dir already exists!" if(-d $dir);
  mkdir $dir;
  die $! if $?;
  for(@projectSubdir){
    mkdir "$dir/$_";
    logmsg "mkdir $dir/$_";
    die $! if $?;
  }
  return $dir;
}

sub addSampleData{
  my($project,$settings)=@_;
  
  my $testdir="$testDir/$$settings{test}";

  # check for files
  my @readsLocal=glob("$testdir/reads/*.fastq.gz");
  my @referenceLocal=glob("$testdir/reference/*.fasta");
  # assembly files?
  my @readsRemote=read_file("$testdir/SRA",{err_mode=>"quiet"});
  my @referenceRemote=read_file("$testdir/reference",{err_mode=>"quiet"});

  my @reads=(@readsLocal,@readsRemote);
  my @reference=_uniq(@referenceLocal,@referenceRemote);

  die "ERROR: no reads found in $testdir\n" if(!@reads);
  die "ERROR: no reference found in $testdir" if(!@reference);
  chomp(@reads,@reference);

  # get the reads
  for(@reads){
    my ($r,$realname)=split(/\t/,$_);
    local $$settings{'add-reads'}=$r;
    my $path=addReads($project,$settings);
    move($path,"$project/reads/$realname.fastq.gz");
    die "ERROR could not rename $path to $project/reads/$realname.fastq.gz: $!" if $?;
  }

  # get the reference
  local $$settings{'change-reference'}=$reference[0];
  logmsg "Using ".$$settings{'change-reference'}." as the reference genome";
  changeReference($project,$settings);

  # assemblies?
  copy($_,"$project/asm/") for(@reference);

  logmsg "Done! To test these data, please execute\n launch_set.pl $project";
  return 1;
}

sub is_project{
  my($project,$settings)=@_;
  my $dirs="";
  for(@projectSubdir){
    $dirs.="$project/$_ " if(!-e "$project/$_");
  }

  if($dirs){
    return (0,$dirs) if wantarray;
    return 0;
  }
  return (1,"") if wantarray;
  return 1;
}

sub addReads{
  my($project,$settings)=@_;<commit_msg>moved all downloading-the-test-dataset functionality out of this script<commit_after>#!/usr/bin/env perl
# Manage a Lyve-SET project
# Author: <NAME> <<EMAIL>>

use strict;
use warnings;
use Data::Dumper;
use Getopt::Long;
use File::Spec;
use Cwd qw/realpath/;
use File::Basename qw/basename/;
use File::Copy qw/copy move/;
use File::Slurp qw/read_file/;
use Bio::Perl;

use FindBin;
use lib "$FindBin::RealBin/../lib";
$ENV{PATH}="$ENV{PATH}:$FindBin::RealBin/../lib/edirect";

# The directories a project should have
my @projectSubdir=qw(vcf vcf/unfiltered msa bam reads reference tmp asm log);

# get project test data
my $testDir="$FindBin::RealBin/../testdata";
my @test=_uniq(map({basename($_) if(-d $_)} glob("$testDir/*")));
my $testdata=join(", ",@test);

sub logmsg{local $0=basename $0; print STDERR "$0: @_\n";}
exit main();

sub main{
  my $settings={};
  GetOptions($settings,qw(help create test=s delete add-reads=s remove-reads=s add-assembly=s remove-assembly=s change-reference=s)) or die $!;
  die usage() if($$settings{help});
  die "ERROR: need a SET project\n".usage() if(!@ARGV);
  my $project=shift(@ARGV);
  $$settings{create}=1 if($$settings{test});

  createProjectDir($project,$settings) if($$settings{create});
  if(!is_project($project,$settings)){
    my(undef,$dirs)=is_project($project,$settings);
    die "$project is not a SET project! Missing $dirs";
  }
  $$settings{tempdir}||="$project/tmp";

  addSampleData($project,$settings) if($$settings{test});
  addReads($project,$settings) if($$settings{'add-reads'});
  addAssembly($project,$settings) if($$settings{'add-assembly'});
  changeReference($project,$settings) if($$settings{'change-reference'});
  removeAssembly($project,$settings) if($$settings{'remove-assembly'});
  removeReads($project,$settings) if($$settings{'remove-reads'});
  deleteProject($project,$settings) if($$settings{delete});
  removeGenome($project,$settings) if($$settings{'remove-genome'});

  return 0;
}

sub createProjectDir{
  my($dir,$settings)=@_;
  die "ERROR: directory $dir already exists!" if(-d $dir);
  mkdir $dir;
  die $! if $?;
  for(@projectSubdir){
    mkdir "$dir/$_";
    logmsg "mkdir $dir/$_";
    die $! if $?;
  }
  return $dir;
}

sub addSampleData{
  my($project,$settings)=@_;
  
  my $testdir="$testDir/$$settings{test}";

  # check for files
  my @reads=glob("$testdir/reads/*.fastq.gz");
  my @reference=glob("$testdir/reference/*.fasta");
  my @assembly=glob("$testdir/asm/*.fasta");

  die "ERROR: no reads found in $testdir\n" if(!@reads);
  die "ERROR: no reference found in $testdir" if(!@reference);
  logmsg "WARNING: no assemblies found in $testdir" if(!@assembly);
  chomp(@reads,@reference,@assembly);

  # get the reads
  for(@reads){
    local $$settings{'add-reads'}=$_;
    my $path=addReads($project,$settings);
  }

  # add assemblies
  for(@assembly){
    local $$settings{'add-assembly'}=$_;
    addAssembly($project,$settings);
  }

  # get the reference
  local $$settings{'change-reference'}=$reference[0];
  logmsg "Using ".$$settings{'change-reference'}." as the reference genome";
  changeReference($project,$settings);

  logmsg "Done! To test these data, please execute\n launch_set.pl $project";
  return 1;
}

sub is_project{
  my($project,$settings)=@_;
  my $dirs="";
  for(@projectSubdir){
    $dirs.="$project/$_ " if(!-e "$project/$_");
  }

  if($dirs){
    return (0,$dirs) if wantarray;
    return 0;
  }
  return (1,"") if wantarray;
  return 1;
}

sub addReads{
  my($project,$settings)=@_;
{'fraction_non_alphanumeric': 0.19655172413793104, 'fraction_numerical': 0.0030344827586206895, 'mean_word_length': 5.382922535211268, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1319054', 'n_tokens_mistral': 2589, 'n_tokens_neox': 2513, 'n_words': 551}