<commit_before>#!/usr/bin/env perl

package exc;

=head1 NAME


=head1 DESCRIPTION


=head1 SYNOPSIS


=head1 OPTIONS

=over

=item --create-config

Create a config file with default settings for user customization.

=item -c|--config

Use user customized config file. Superseeds default config.

=item --debug

Verbose debug messages.

=item -V|--version

Display version.

=item -h|--help

Display this help.

=back

=head1 CHANGELOG

see git log.

=head1 TODO

=head1 CODE

=cut

#-----------------------------------------------------------------------------#
# Modules

# core
use strict;
use warnings;
no warnings 'qw';

use Carp;
use Getopt::Long qw(:config no_ignore_case bundling);
use Pod::Usage;
use Log::Log4perl qw(:no_extra_logdie_message);
use Log::Log4perl::Level;

use Data::Dumper;
$Data::Dumper::Sortkeys = 1;

use FindBin qw($RealBin $Script);
use lib "$RealBin/../lib/";

use File::Basename;
use File::Copy;
use File::Path qw(make_path);
use File::Find;

# additional modules
use Cfg;
use Fasta::Parser;
use Fastq::Parser;
use Bowtie2;
use Sam::Parser;
use Cwd;

#-----------------------------------------------------------------------------#
# Globals

our $VERSION = 0.02;

our $ID = 'exc';

# get a logger
my $L = Log::Log4perl::get_logger();

my $log_cfg = 'log4perl.rootLogger                     = INFO, Screen
log4perl.appender.Screen                = Log::Log4perl::Appender::Screen
log4perl.appender.Screen.stderr         = 1
log4perl.appender.Screen.layout         = PatternLayout
log4perl.appender.Screen.layout.ConversionPattern = [%d{yy-MM-dd HH:mm:ss}] ['.$ID.'] %m%n
';

Log::Log4perl->init( \$log_cfg );


#-----------------------------------------------------------------------------#
# GetOptions

# opt: multi-params need to be initiated with ARRAYREF!
my %opt = (
    config => [],
    );

# Setup defaults
my %def = (
    );


GetOptions( # use %opt (Cfg) as defaults
	\%opt, qw(
		version|V!
		debug|D!
		help|h!
		config|c=s
		reads|1=s
		mates|2=s
		insert_size|insert-size|isize|s=i
		border=i
		in|i=s
		out|o=s
		coverage=s
		threads=i
	)
) or $L->logcroak('Failed to "GetOptions"');

# help
$opt{help} && pod2usage(1);

# version
if($opt{version}){
	print "$VERSION\n"; 
	exit 0;
}


##----------------------------------------------------------------------------##
# Config

my %cfg;

# core
my $core_cfg = "$RealBin/../".basename($Script, qw(.pl)).".cfg";

if(-e $core_cfg){
    $opt{core_config} = File::Spec->rel2abs($core_cfg);
    %cfg = (%cfg, Cfg->Read($opt{core_config}, $ID));
}



# read all configs
if (@{$opt{config}}){
    foreach my $cfg ( @{$opt{config}} ){
	# $L->info("Reading config $cfg");
	$cfg=File::Spec->rel2abs($cfg);
	%cfg = (%cfg, Cfg->Read($cfg, $ID));
    }
}

# create template for user cfg
if(defined $opt{create_config}){
    pod2usage(-msg => 'To many arguments', -exitval=>1) if @ARGV > 1;
    my $user_cfg = Cfg->Copy($core_cfg, $opt{create_config}) or $L->logdie("Creating config failed: $!");
    $L->info("Created config file: $user_cfg");
    exit 0;
}


# Merge opt and cfg
%opt = (%cfg, %opt);


# debug level
$opt{quiet} && $L->level($WARN);
$opt{debug} && $L->level($DEBUG);

$L->debug(Dumper(\%opt));



##----------------------------------------------------------------------------##
# required	
for(qw(in out insert_size mates reads)){
    if(ref $opt{$_} eq 'ARRAY'){
	pod2usage("required: --$_") unless @{$opt{$_}}
    }else{
	pod2usage("required: --$_") unless defined ($opt{$_})
    }
};


## check if the border was set, otherwise use 2x insert_size as default value
unless (exists $opt{border} && $opt{border} > 0)
{
    $opt{border} = $opt{insert_size}*2;
}


#-----------------------------------------------------------------------------#
# MAIN

$L->info('Extend contig script');

$L->info("Generating 5' and 3' ends of contigs");

#### first generate a FASTA file for 5' and 3' ends from input file
my $fasta_in = Fasta::Parser->new(
    file => $opt{in}
    );
my $fasta_contig_ends=$opt{in}.'_contig_ends';
my $fasta_out = Fasta::Parser->new(
    file => $fasta_contig_ends,
    mode => '>'                    # overwrite an existing file
    );
# loop through the FASTA file and generate a set of contig ends
my %contig_ends_seen = ();
my %filehandles = ();
while (my $contig=$fasta_in->next_seq())
{
    # filter for a minimum length
    next unless (length($contig->seq()) > $opt{min_seq_length});

    # store 5' end
    store_sequence_and_create_folder(name => $contig->id()."_5prime", seq => substr($contig->seq(), 0, $opt{border}), file_out => $fasta_out, seen_names => \%contig_ends_seen, filehandle => \%filehandles);
    # store 3' end
    store_sequence_and_create_folder(name => $contig->id()."_3prime", seq => substr($contig->seq(), -$opt{border}), file_out => $fasta_out, seen_names => \%contig_ends_seen, filehandle => \%filehandles);
}

# create a sequence store for each border combination
# therefore I need to retrieve a list of all sequence names
my @seq_names = sort keys %filehandles;
# now prepare each combination
for (my $i=0; $i < @seq_names - 1; $i++)
{
    for (my $j=$i+1; $j < @seq_names; $j++)
    {
	my $mergename = join("_", ("merged", $seq_names[$i], $seq_names[$j]));
	
	store_sequence_and_create_folder(name => $mergename, file_out => $fasta_out, seen_names => \%contig_ends_seen, filehandle => \%filehandles);
    }
}

my $bowtie2 = Bowtie2->new(
    path => $opt{bowtie2_path},
);

$L->info("Building bowtie2 index");

$bowtie2->build($fasta_contig_ends);

##
$L->debug("FILEHANDLES:", Dumper(\%filehandles));

$L->info("Running bowtie2");

$bowtie2->run(
    @{$opt{bowtie2_params}},
    "-1" => $opt{reads},
    "-2" => $opt{mates},
    "-x" => $fasta_contig_ends,
    "-p" => $opt{threads} || 1,
#    "--all" => '',           # generate all alignments
);

# read output on the fly
my $sp = Sam::Parser->new(
  fh => $bowtie2->stdout,
);

# print sequences of read pairs with both reads mapped
while( my ($aln1, $aln2) = $sp->next_pair() ){<commit_msg>[exc] now deleting empty folders in merged_ass folder<commit_after>#!/usr/bin/env perl

package exc;

=head1 NAME


=head1 DESCRIPTION


=head1 SYNOPSIS


=head1 OPTIONS

=over

=item --create-config

Create a config file with default settings for user customization.

=item -c|--config

Use user customized config file. Superseeds default config.

=item --debug

Verbose debug messages.

=item -V|--version

Display version.

=item -h|--help

Display this help.

=back

=head1 CHANGELOG

see git log.

=head1 TODO

=head1 CODE

=cut

#-----------------------------------------------------------------------------#
# Modules

# core
use strict;
use warnings;
no warnings 'qw';

use Carp;
use Getopt::Long qw(:config no_ignore_case bundling);
use Pod::Usage;
use Log::Log4perl qw(:no_extra_logdie_message);
use Log::Log4perl::Level;

use Data::Dumper;
$Data::Dumper::Sortkeys = 1;

use FindBin qw($RealBin $Script);
use lib "$RealBin/../lib/";

use File::Basename;
use File::Copy;
use File::Path qw(make_path);
use File::Find;

# additional modules
use Cfg;
use Fasta::Parser;
use Fastq::Parser;
use Bowtie2;
use Sam::Parser;
use Cwd;

#-----------------------------------------------------------------------------#
# Globals

our $VERSION = 0.02;

our $ID = 'exc';

# get a logger
my $L = Log::Log4perl::get_logger();

my $log_cfg = 'log4perl.rootLogger                     = INFO, Screen
log4perl.appender.Screen                = Log::Log4perl::Appender::Screen
log4perl.appender.Screen.stderr         = 1
log4perl.appender.Screen.layout         = PatternLayout
log4perl.appender.Screen.layout.ConversionPattern = [%d{yy-MM-dd HH:mm:ss}] ['.$ID.'] %m%n
';

Log::Log4perl->init( \$log_cfg );


#-----------------------------------------------------------------------------#
# GetOptions

# opt: multi-params need to be initiated with ARRAYREF!
my %opt = (
    config => [],
    );

# Setup defaults
my %def = (
    );


GetOptions( # use %opt (Cfg) as defaults
	\%opt, qw(
		version|V!
		debug|D!
		help|h!
		config|c=s
		reads|1=s
		mates|2=s
		insert_size|insert-size|isize|s=i
		border=i
		in|i=s
		out|o=s
		coverage=s
		threads=i
	)
) or $L->logcroak('Failed to "GetOptions"');

# help
$opt{help} && pod2usage(1);

# version
if($opt{version}){
	print "$VERSION\n"; 
	exit 0;
}


##----------------------------------------------------------------------------##
# Config

my %cfg;

# core
my $core_cfg = "$RealBin/../".basename($Script, qw(.pl)).".cfg";

if(-e $core_cfg){
    $opt{core_config} = File::Spec->rel2abs($core_cfg);
    %cfg = (%cfg, Cfg->Read($opt{core_config}, $ID));
}



# read all configs
if (@{$opt{config}}){
    foreach my $cfg ( @{$opt{config}} ){
	# $L->info("Reading config $cfg");
	$cfg=File::Spec->rel2abs($cfg);
	%cfg = (%cfg, Cfg->Read($cfg, $ID));
    }
}

# create template for user cfg
if(defined $opt{create_config}){
    pod2usage(-msg => 'To many arguments', -exitval=>1) if @ARGV > 1;
    my $user_cfg = Cfg->Copy($core_cfg, $opt{create_config}) or $L->logdie("Creating config failed: $!");
    $L->info("Created config file: $user_cfg");
    exit 0;
}


# Merge opt and cfg
%opt = (%cfg, %opt);


# debug level
$opt{quiet} && $L->level($WARN);
$opt{debug} && $L->level($DEBUG);

$L->debug(Dumper(\%opt));



##----------------------------------------------------------------------------##
# required	
for(qw(in out insert_size mates reads)){
    if(ref $opt{$_} eq 'ARRAY'){
	pod2usage("required: --$_") unless @{$opt{$_}}
    }else{
	pod2usage("required: --$_") unless defined ($opt{$_})
    }
};


## check if the border was set, otherwise use 2x insert_size as default value
unless (exists $opt{border} && $opt{border} > 0)
{
    $opt{border} = $opt{insert_size}*2;
}


#-----------------------------------------------------------------------------#
# MAIN

$L->info('Extend contig script');

$L->info("Generating 5' and 3' ends of contigs");

#### first generate a FASTA file for 5' and 3' ends from input file
my $fasta_in = Fasta::Parser->new(
    file => $opt{in}
    );
my $fasta_contig_ends=$opt{in}.'_contig_ends';
my $fasta_out = Fasta::Parser->new(
    file => $fasta_contig_ends,
    mode => '>'                    # overwrite an existing file
    );
# loop through the FASTA file and generate a set of contig ends
my %contig_ends_seen = ();
my %filehandles = ();
while (my $contig=$fasta_in->next_seq())
{
    # filter for a minimum length
    next unless (length($contig->seq()) > $opt{min_seq_length});

    # store 5' end
    store_sequence_and_create_folder(name => $contig->id()."_5prime", seq => substr($contig->seq(), 0, $opt{border}), file_out => $fasta_out, seen_names => \%contig_ends_seen, filehandle => \%filehandles);
    # store 3' end
    store_sequence_and_create_folder(name => $contig->id()."_3prime", seq => substr($contig->seq(), -$opt{border}), file_out => $fasta_out, seen_names => \%contig_ends_seen, filehandle => \%filehandles);
}

# create a sequence store for each border combination
# therefore I need to retrieve a list of all sequence names
my @seq_names = sort keys %filehandles;
# now prepare each combination
for (my $i=0; $i < @seq_names - 1; $i++)
{
    for (my $j=$i+1; $j < @seq_names; $j++)
    {
	my $mergename = join("_", ("merged", $seq_names[$i], $seq_names[$j]));
	
	store_sequence_and_create_folder(name => $mergename, file_out => $fasta_out, seen_names => \%contig_ends_seen, filehandle => \%filehandles);
    }
}

#delete empty directories
my $dir = './merged_ass';
my @dirs;
find(\&get_dir,$dir);

rmdir($_) for(reverse(@dirs));

sub get_dir{
  push(@dirs,$File::Find::name) if(-d $_);
}

my $bowtie2 = Bowtie2->new(
    path => $opt{bowtie2_path},
);

$L->info("Building bowtie2 index");

$bowtie2->build($fasta_contig_ends);

##
$L->debug("FILEHANDLES:", Dumper(\%filehandles));

$L->info("Running bowtie2");

$bowtie2->run(
    @{$opt{bowtie2_params}},
    "-1" => $opt{reads},
    "-2" => $opt{mates},
    "-x" => $fasta_contig_ends,
    "-p" => $opt{threads} || 1,
#    "--all" => '',           # generate all alignments
);

# read output on the fly
my $sp = Sam::Parser->new(
  fh => $bowtie2->stdout,
);

# print sequences of read pairs with both reads mapped
while( my ($aln1, $aln2) = $sp->next_pair() ){
{'fraction_non_alphanumeric': 0.24488124488124488, 'fraction_numerical': 0.010647010647010647, 'mean_word_length': 2.817130353235386, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5141979', 'n_tokens_mistral': 4476, 'n_tokens_neox': 4266, 'n_words': 1054}