<commit_before>#!/usr/bin/perl

use DBI;
use strict;
use CoGeX;
use Roman;
use Data::Dumper;
use Getopt::Long;
use File::Path;

use vars qw($DEBUG $coge $GENOMIC_SEQ_LEN $GO $ERASE);


my ($nt_file, $nt_dir, $org_name, $org_desc, $org_id, $org_restricted, $source_name, $source_desc, $source_link, $source_id, $ds_name, $ds_desc, $ds_link, $ds_version, $ds_id, $chr, $seq_type_name, $seq_type_desc, $seq_type_id, $chr_basename, $add_chr_name, $use_fasta_header, $dsg_name, $dsg_desc, $dsg_version, $dsg_id, $restricted, $db, $user, $pass, $seq_dir);

##Example usage:
#./fasta_genome_loader.pl -org_name "Allenigales" -source_id 24 -ds_name oldsuper2.fasta -ds_version 2 -use_fasta_header -nt ~/projects/genome/data/Selaginella_moellendorffii/pre-v2/oldsuper2.fasta


GetOptions ( "debug=s" => \$DEBUG,
	     "go=s"    => \$GO,
	     "erase|e" => \$ERASE, 
	     "fasta_file|fasta|faa|nt_file|nt=s" => \$nt_file,
	     "fasta_dir|nt_dir|dir=s"=>\$nt_dir,
	     "org_name=s" => \$org_name,
	     "org_desc=s" => \$org_desc,
	     "org_id=s"   => \$org_id,
	     "org_restricted" => \$org_restricted,#set flag to make organism restriced
	     "source_name=s" => \$source_name, # datasource
	     "source_desc=s" => \$source_desc,
	     "source_link=s" => \$source_link,
	     "source_id=s"   => \$source_id,
	     "ds_name=s" => \$ds_name,# datasetid
	     "ds_desc=s" => \$ds_desc,
	     "ds_link=s" => \$ds_link,
	     "ds_version=s" => \$ds_version,
	     "ds_id=s"=>\$ds_id,
	     "dsg_name=s" => \$dsg_name,
	     "dsg_desc=s" => \$dsg_desc,
	     "dsg_version=s" => \$dsg_version,
	     "dsg_id=s"=>\$dsg_id,
	     "restricted=i"=>\$restricted,

	     "chr=s"=>\$chr,
	     "seq_type_name=s" => \$seq_type_name,
	     "seq_type_desc=s" => \$seq_type_desc,
	     "seq_type_id=i"=>\$seq_type_id, # masked50 == id 2
	     "chr_basename=s"=>\$chr_basename,
	     "add_chr_name=s"=>\$add_chr_name,
	     "use_fasta_header"=>\$use_fasta_header,
	     "database|db=s"=>\$db,
	    "user|u=s"=>\$user,
	    "password|pw=s"=>\$pass,
	     "seq_dir|sd=s"=>\$seq_dir,
	   );

$DEBUG = 1 unless defined $DEBUG; # set to '1' to get updates on what's going on
$GO = 1 unless defined $GO; #set to 1 to actually make db calls.
$ds_name = $nt_file unless $ds_name;
$restricted = 0 unless defined $restricted;
print STDERR "Running $0\n";

my $formatdb =  "/usr/bin/formatdb -p F -o T"; #path to blast's formatdb program





my $connstr = "dbi:mysql:dbname=$db;host=localhost;port=3307";
$coge = CoGeX->connect($connstr, $user, $pass );
#$coge->storage->debugobj(new DBIxProfiler());
#$coge->storage->debug(1);



if ($org_name)
  {
    my $org = $coge->resultset("Organism")->find_or_create({name=>$org_name,description=>$org_desc});
    if ($org_restricted)
      {
	$org->restricted(1);
	$org->update;
      }
    $org_id = $org->id;
  }

if ($source_name)
  {
    my $source = $coge->resultset("DataSource")->find_or_create({name=>$source_name,description=>$source_desc, link=>$source_link});
    $source_id = $source->id;
  }


unless ($org_id && $source_id)
  {
    print "Need a valid organism id and data source id in order to create dataset and dataset_group objects.  Loading aborted.\n";
    exit;
  }

if ($seq_type_name)
  {
    my $gst = $coge->resultset('GenomicSequenceType')->find_or_create({name=>$seq_type_name,
								       description=>$seq_type_desc,
								      });
    $seq_type_id = $gst->id;
  }
$seq_type_id = 1 unless $seq_type_id;  #default to unmasked sequence data


my $ds = generate_ds(ds_name => $ds_name,
		     ds_desc => $ds_desc,
		     ds_link => $ds_link,
		     ds_version => $ds_version,
		     ds_id =>$ds_id,
		     source_id=>$source_id,
		     restricted=>$restricted,
		    );

unless ($ds)
  {
    warn "dataset object not initialized.  Exiting.";
    exit;    
  }


$dsg_version = $ds->version unless $dsg_version;
my $dsg = generate_dsg(name=>$dsg_name,
		       desc=>$dsg_desc,
		       version=>$dsg_version,
		       dsg_id=>$dsg_id,
		       org_id=>$org_id,
		       gst_id=>$seq_type_id,
		       restricted=>$restricted,
		       );
#link dataset and dataset_group

$coge->resultset('DatasetConnector')->find_or_create({dataset_id=>$ds->id, dataset_group_id=>$dsg->id});


if ($ERASE)
  {
    print "Clearing database of entries associated with ".$ds->name.". . .";
    $ds->delete();
    $dsg->delete();
    print "finished!\n";
    exit;
  }

process_nt (file=>$nt_file, ds=>$ds, dsg=>$dsg, dir=>$nt_dir, chr=>$chr) if $nt_file || $nt_dir;
print "dataset_id: ".$ds->id,"\n";
print "dataset_group_id: ".$dsg->id,"\n";




sub add_location<commit_msg>updated parsing of nt_file name if used as the ds_name<commit_after>#!/usr/bin/perl

use DBI;
use strict;
use CoGeX;
use Roman;
use Data::Dumper;
use Getopt::Long;
use File::Path;

use vars qw($DEBUG $coge $GENOMIC_SEQ_LEN $GO $ERASE);


my ($nt_file, $nt_dir, $org_name, $org_desc, $org_id, $org_restricted, $source_name, $source_desc, $source_link, $source_id, $ds_name, $ds_desc, $ds_link, $ds_version, $ds_id, $chr, $seq_type_name, $seq_type_desc, $seq_type_id, $chr_basename, $add_chr_name, $use_fasta_header, $dsg_name, $dsg_desc, $dsg_version, $dsg_id, $restricted, $db, $user, $pass, $seq_dir);

##Example usage:
#./fasta_genome_loader.pl -org_name "Allenigales" -source_id 24 -ds_name oldsuper2.fasta -ds_version 2 -use_fasta_header -nt ~/projects/genome/data/Selaginella_moellendorffii/pre-v2/oldsuper2.fasta


GetOptions ( "debug=s" => \$DEBUG,
	     "go=s"    => \$GO,
	     "erase|e" => \$ERASE, 
	     "fasta_file|fasta|faa|nt_file|nt=s" => \$nt_file,
	     "fasta_dir|nt_dir|dir=s"=>\$nt_dir,
	     "org_name=s" => \$org_name,
	     "org_desc=s" => \$org_desc,
	     "org_id=s"   => \$org_id,
	     "org_restricted" => \$org_restricted,#set flag to make organism restriced
	     "source_name=s" => \$source_name, # datasource
	     "source_desc=s" => \$source_desc,
	     "source_link=s" => \$source_link,
	     "source_id=s"   => \$source_id,
	     "ds_name=s" => \$ds_name,# datasetid
	     "ds_desc=s" => \$ds_desc,
	     "ds_link=s" => \$ds_link,
	     "ds_version=s" => \$ds_version,
	     "ds_id=s"=>\$ds_id,
	     "dsg_name=s" => \$dsg_name,
	     "dsg_desc=s" => \$dsg_desc,
	     "dsg_version=s" => \$dsg_version,
	     "dsg_id=s"=>\$dsg_id,
	     "restricted=i"=>\$restricted,

	     "chr=s"=>\$chr,
	     "seq_type_name=s" => \$seq_type_name,
	     "seq_type_desc=s" => \$seq_type_desc,
	     "seq_type_id=i"=>\$seq_type_id, # masked50 == id 2
	     "chr_basename=s"=>\$chr_basename,
	     "add_chr_name=s"=>\$add_chr_name,
	     "use_fasta_header"=>\$use_fasta_header,
	     "database|db=s"=>\$db,
	    "user|u=s"=>\$user,
	    "password|pw=s"=>\$pass,
	     "seq_dir|sd=s"=>\$seq_dir,
	   );

$DEBUG = 1 unless defined $DEBUG; # set to '1' to get updates on what's going on
$GO = 1 unless defined $GO; #set to 1 to actually make db calls.
($ds_name) = $nt_file=~ /([^\/]*)$/ unless ($ds_name);


$restricted = 0 unless defined $restricted;
print STDERR "Running $0\n";

my $formatdb =  "/usr/bin/formatdb -p F -o T"; #path to blast's formatdb program





my $connstr = "dbi:mysql:dbname=$db;host=localhost;port=3307";
$coge = CoGeX->connect($connstr, $user, $pass );
#$coge->storage->debugobj(new DBIxProfiler());
#$coge->storage->debug(1);



if ($org_name)
  {
    my $org = $coge->resultset("Organism")->find_or_create({name=>$org_name,description=>$org_desc});
    if ($org_restricted)
      {
	$org->restricted(1);
	$org->update;
      }
    $org_id = $org->id;
  }

if ($source_name)
  {
    my $source = $coge->resultset("DataSource")->find_or_create({name=>$source_name,description=>$source_desc, link=>$source_link});
    $source_id = $source->id;
  }


unless ($org_id && $source_id)
  {
    print "Need a valid organism id and data source id in order to create dataset and dataset_group objects.  Loading aborted.\n";
    exit;
  }

if ($seq_type_name)
  {
    my $gst = $coge->resultset('GenomicSequenceType')->find_or_create({name=>$seq_type_name,
								       description=>$seq_type_desc,
								      });
    $seq_type_id = $gst->id;
  }
$seq_type_id = 1 unless $seq_type_id;  #default to unmasked sequence data


my $ds = generate_ds(ds_name => $ds_name,
		     ds_desc => $ds_desc,
		     ds_link => $ds_link,
		     ds_version => $ds_version,
		     ds_id =>$ds_id,
		     source_id=>$source_id,
		     restricted=>$restricted,
		    );

unless ($ds)
  {
    warn "dataset object not initialized.  Exiting.";
    exit;    
  }


$dsg_version = $ds->version unless $dsg_version;
my $dsg = generate_dsg(name=>$dsg_name,
		       desc=>$dsg_desc,
		       version=>$dsg_version,
		       dsg_id=>$dsg_id,
		       org_id=>$org_id,
		       gst_id=>$seq_type_id,
		       restricted=>$restricted,
		       );
#link dataset and dataset_group

$coge->resultset('DatasetConnector')->find_or_create({dataset_id=>$ds->id, dataset_group_id=>$dsg->id});


if ($ERASE)
  {
    print "Clearing database of entries associated with ".$ds->name.". . .";
    $ds->delete();
    $dsg->delete();
    print "finished!\n";
    exit;
  }
print "dataset_id: ".$ds->id,"\n";
print "dataset_group_id: ".$dsg->id,"\n\n\n";

process_nt (file=>$nt_file, ds=>$ds, dsg=>$dsg, dir=>$nt_dir, chr=>$chr) if $nt_file || $nt_dir;
print "dataset_id: ".$ds->id,"\n";
print "dataset_group_id: ".$dsg->id,"\n";




sub add_location
{'fraction_non_alphanumeric': 0.18229501177478055, 'fraction_numerical': 0.004709912224363092, 'mean_word_length': 3.6183885318833418, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 181, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2497687', 'n_tokens_mistral': 4073, 'n_tokens_neox': 3735, 'n_words': 676}