<commit_before>package Transposome::SeqUtil;

use 5.012;
use Moose;
use Moose::Util::TypeConstraints;
use namespace::autoclean;
use DB_File;
use vars qw($DB_BTREE &R_DUP);  
use Transposome::SeqIO;

with 'MooseX::Log::Log4perl',
     'Transposome::Role::File',
     'Transposome::Role::Types';

=head1 NAME

Transposome::SeqUtil - Utilities for handling Fasta/q sequence files.

=head1 VERSION

Version 0.01

=cut

our $VERSION = '0.01';

=head1 SYNOPSIS

For storing all sequences:

    use Transposome::SeqUtil;

    my $sequtil = Transposome::SeqUtil->new( file      => 'myseqs.fas',
                                             in_memory => 1,
                                           );
    my ($seqs, $seqct) = $sequtil->store_seq;

    ...

For sampling a sequence file:

use Transposome::SeqUtil;

    my $sequtil = Transposome::SeqUtil->new( file        => 'myseqs.fas',
                                             sample_size => 500_000,
                                           );

    $sequtil->sample_seq;

=cut

subtype 'ModNum'
    => as 'Num'
    => where { /\_/ || /\d+/ };

coerce 'ModNum',
    from 'Str',
    via { $_ =~ s/\_//g; 0+$_ };

has 'in_memory' => (
    is         => 'ro',
    isa        => 'Bool',
    predicate  => 'has_in_memory',
    lazy       => 1,
    default    => 0,
    );

has 'sample_size' => (
    is        => 'ro',
    isa       => 'ModNum',
    predicate => 'has_sample',
    coerce    => 1,
    );

has 'seed' => (
    is        => 'ro',
    isa       => 'Int',
    predicate => 'has_seed',
    lazy      => 1,
    default   => 11,
    );

has 'no_store' => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_no_store',
    lazy      => 1,
    default   => 0,
    );

=head1 METHODS

=head2 store_seq

 Title   : store_seq
 Usage   : my ($seqs, $seq_ct) = $seq_store->store_seq;
          
 Function: Take a Fasta or Fastq file and return a reference
           to a data structure containing the sequences, along
           with the total sequence count.

                                                               Return_type
Returns : In order, 1) a hash containing the id, sequence      HashRef 
                        mappings for each Fasta/q record
                     2) the sequence count                     Scalar

                                                               Arg_type
 Args    : A sequence file                                     Scalar


=cut

sub store_seq {
    my ($self) = @_;
 
    my %seqhash;
    unless ($self->in_memory) {
        $DB_BTREE->{cachesize} = 100000;
        $DB_BTREE->{flags} = R_DUP;
        my $seq_dbm = "transposome_seqstore.dbm";
        unlink $seq_dbm if -e $seq_dbm;
        tie %seqhash, 'DB_File', $seq_dbm, O_RDWR|O_CREAT, 0666, $DB_BTREE
            or die "\n[ERROR]: Could not open DBM file $seq_dbm: $!\n";
    }

    if (-e $self->file) {
	my $filename = $self->file->relative;
	my $seqio = Transposome::SeqIO->new( file => $filename );
	my $fh = $seqio->get_fh;
	while (my $seq = $seqio->next_seq($fh)) {
	    $self->inc_counter if $seq->has_seq;
	    $seqhash{$seq->get_id} = $seq->get_seq;
	}
	return (\%seqhash, $self->counter);
    }
}

=head2 sample_seq

 Title   : sample_seq
 Usage   : my ($seqs, $seq_ct) = $sequtil->sample_seq;
          
 Function: Take a Fasta or Fastq file and return a reference
           to a data structure containing the sequences, along
           with the total sequence count.

                                                               Return_type
 Returns : In order, 1) a hash containing the id, sequence     HashRef 
                        mappings for each Fasta/q record
                     2) the sequence count                     Scalar

           If the object is created with the argument
           "no_store => 1" then calling this method causes
           the sequences to simply be written to STDOUT.
                                                               Arg_type
 Args    : A sequence file                                     Scalar

=cut

sub sample_seq {
    my ($self) = @_;

    my $filename = $self->file->relative;
    my $k = $self->sample_size;
    my $seed = $self->seed;
    my $n = 0;
    my @sample;
    my %seqhash;

    my $seqio_fa = Transposome::SeqIO->new( file => $filename );
    my $seqfh = $seqio_fa->get_fh;

    srand($seed);
    while (my $seq = $seqio_fa->next_seq($seqfh)) {
	$n++;
	push @sample, {$seq->get_id => $seq->get_seq};
	last if $n == $k;
    }

    while (my $seq = $seqio_fa->next_seq($seqfh)) {
	my $i = int rand $n++;
	if ($i < scalar @sample) {
	    $sample[$i] = {$seq->get_id => $seq->get_seq};
	}
    }

    for my $seq (@sample) {
	for my $h (keys %$seq) {
	    if ($self->no_store) {
		say join "\n", ">".$h, $seq->{$h};
	    }
	    else {<commit_msg>modifications for new seqio next_seq method<commit_after>package Transposome::SeqUtil;

use 5.012;
use Moose;
use Moose::Util::TypeConstraints;
use namespace::autoclean;
use DB_File;
use vars qw($DB_BTREE &R_DUP);  
use Transposome::SeqIO;

with 'MooseX::Log::Log4perl',
     'Transposome::Role::File',
     'Transposome::Role::Types';

=head1 NAME

Transposome::SeqUtil - Utilities for handling Fasta/q sequence files.

=head1 VERSION

Version 0.01

=cut

our $VERSION = '0.01';

=head1 SYNOPSIS

For storing all sequences:

    use Transposome::SeqUtil;

    my $sequtil = Transposome::SeqUtil->new( file      => 'myseqs.fas',
                                             in_memory => 1,
                                           );
    my ($seqs, $seqct) = $sequtil->store_seq;

    ...

For sampling a sequence file:

use Transposome::SeqUtil;

    my $sequtil = Transposome::SeqUtil->new( file        => 'myseqs.fas',
                                             sample_size => 500_000,
                                           );

    $sequtil->sample_seq;

=cut

subtype 'ModNum'
    => as 'Num'
    => where { /\_/ || /\d+/ };

coerce 'ModNum',
    from 'Str',
    via { $_ =~ s/\_//g; 0+$_ };

has 'in_memory' => (
    is         => 'ro',
    isa        => 'Bool',
    predicate  => 'has_in_memory',
    lazy       => 1,
    default    => 0,
    );

has 'sample_size' => (
    is        => 'ro',
    isa       => 'ModNum',
    predicate => 'has_sample',
    coerce    => 1,
    );

has 'seed' => (
    is        => 'ro',
    isa       => 'Int',
    predicate => 'has_seed',
    lazy      => 1,
    default   => 11,
    );

has 'no_store' => (
    is        => 'ro',
    isa       => 'Bool',
    predicate => 'has_no_store',
    lazy      => 1,
    default   => 0,
    );

=head1 METHODS

=head2 store_seq

 Title   : store_seq
 Usage   : my ($seqs, $seq_ct) = $seq_store->store_seq;
          
 Function: Take a Fasta or Fastq file and return a reference
           to a data structure containing the sequences, along
           with the total sequence count.

                                                               Return_type
Returns : In order, 1) a hash containing the id, sequence      HashRef 
                        mappings for each Fasta/q record
                     2) the sequence count                     Scalar

                                                               Arg_type
 Args    : A sequence file                                     Scalar


=cut

sub store_seq {
    my ($self) = @_;
 
    my %seqhash;
    unless ($self->in_memory) {
        $DB_BTREE->{cachesize} = 100000;
        $DB_BTREE->{flags} = R_DUP;
        my $seq_dbm = "transposome_seqstore.dbm";
        unlink $seq_dbm if -e $seq_dbm;
        tie %seqhash, 'DB_File', $seq_dbm, O_RDWR|O_CREAT, 0666, $DB_BTREE
            or die "\n[ERROR]: Could not open DBM file $seq_dbm: $!\n";
    }

    if (-e $self->file) {
	my $filename = $self->file->relative;
	my $seqio = Transposome::SeqIO->new( file => $filename );
	while (my $seq = $seqio->next_seq) {
	    $self->inc_counter if $seq->has_seq;
	    $seqhash{$seq->get_id} = $seq->get_seq;
	}
	return (\%seqhash, $self->counter);
    }
}

=head2 sample_seq

 Title   : sample_seq
 Usage   : my ($seqs, $seq_ct) = $sequtil->sample_seq;
          
 Function: Take a Fasta or Fastq file and return a reference
           to a data structure containing the sequences, along
           with the total sequence count.

                                                               Return_type
 Returns : In order, 1) a hash containing the id, sequence     HashRef 
                        mappings for each Fasta/q record
                     2) the sequence count                     Scalar

           If the object is created with the argument
           "no_store => 1" then calling this method causes
           the sequences to simply be written to STDOUT.
                                                               Arg_type
 Args    : A sequence file                                     Scalar

=cut

sub sample_seq {
    my ($self) = @_;

    my $filename = $self->file->relative;
    my $k = $self->sample_size;
    my $seed = $self->seed;
    my $n = 0;
    my @sample;
    my %seqhash;

    my $seqio_fa = Transposome::SeqIO->new( file => $filename );

    srand($seed);
    while (my $seq = $seqio_fa->next_seq) {
	$n++;
	push @sample, {$seq->get_id => $seq->get_seq};
	last if $n == $k;
    }

    while (my $seq = $seqio_fa->next_seq($seqfh)) {
	my $i = int rand $n++;
	if ($i < scalar @sample) {
	    $sample[$i] = {$seq->get_id => $seq->get_seq};
	}
    }

    for my $seq (@sample) {
	for my $h (keys %$seq) {
	    if ($self->no_store) {
		say join "\n", ">".$h, $seq->{$h};
	    }
	    else {
{'fraction_non_alphanumeric': 0.1296643735668126, 'fraction_numerical': 0.010214717531790703, 'mean_word_length': 1.6309295311214698, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2729954', 'n_tokens_mistral': 3198, 'n_tokens_neox': 3027, 'n_words': 839}