<commit_before>#!/usr/bin/env perl

=head1 NAME

generate-names.pl - generate a global index of feature names

=head1 USAGE

  generate-names.pl                        \
      [ --out <output directory> ]         \
      [ --thresh <threshold> ]             \
      [ --verbose ]

=head1 OPTIONS

=over 4

=item --out <directory>

Data directory to process.  Default 'data/'.

=item --tracks <trackname>[,...]

Comma-separated list of which tracks to include in the names index.  If
not passed, all tracks are indexed.

=item --thresh <threshold>

Optional LazyPatricia chunking threshold, in bytes.  Default 100kb.  See
L<LazyPatricia> for details.

=item --completionLimit <number>

Maximum number of completions to store for a given prefix.  Default 50.

=item --verbose

Print more progress messages.

=item --help | -h | -?

Print a usage message.

=back

=cut

use strict;
use warnings;

use FindBin qw($Bin);
use lib "$Bin/../src/perl5";
use JBlibs;

use Fcntl ":flock";
use File::Spec::Functions;
use Getopt::Long;
use IO::File;
use Pod::Usage;

use JSON 2;


use LazyPatricia;

use Bio::JBrowse::HashStore;
use GenomeDB;

use Data::Dumper;
$Data::Dumper::Indent = 1;

my %trackHash;
my @includedTrackNames;
my @tracksWithNames;

# TODO: change to hash-based storage, pre-generate prefixes

my $outDir = "data";
my $thresh = 100 * 2**10;
my $verbose = 0;
my $incremental;
my $help;
my $max_completions = 50;
GetOptions("dir|out=s" => \$outDir,
           "thresh=i" => \$thresh,
           "completionLimit=i" => \$max_completions,
           "verbose+" => \$verbose,
           "add"      => \$incremental,
           'tracks=s' => \@includedTrackNames,
           "help|h|?" => \$help) or pod2usage();

my %includedTrackNames = map { $_ => 1 }
                         map { split ',', $_ }
                         @includedTrackNames;

pod2usage( -verbose => 2 ) if $help;

unless (-d $outDir) {
    die <<OUTDIR;
Can't find directory "$outDir".
Run this program from a different working directory,
or specify the location of the output directory with
the --dir command line option.
OUTDIR
}

my $gdb = GenomeDB->new( $outDir );
my $nameStore = Bio::JBrowse::HashStore->open( dir => catdir( $outDir, "names" ) );

my @refSeqs  = @{ $gdb->refSeqs   };
my @tracks   = grep { !%includedTrackNames || $includedTrackNames{ $_->{label} } }
               @{ $gdb->trackList };

if( $verbose ) {
    print STDERR "Tracks:\n".join('', map "    $_->{label}\n", @tracks );
}


# clear out old data<commit_msg>fix generate-names.pl behavior when no tracks defined<commit_after>#!/usr/bin/env perl

=head1 NAME

generate-names.pl - generate a global index of feature names

=head1 USAGE

  generate-names.pl                        \
      [ --out <output directory> ]         \
      [ --thresh <threshold> ]             \
      [ --verbose ]

=head1 OPTIONS

=over 4

=item --out <directory>

Data directory to process.  Default 'data/'.

=item --tracks <trackname>[,...]

Comma-separated list of which tracks to include in the names index.  If
not passed, all tracks are indexed.

=item --thresh <threshold>

Optional LazyPatricia chunking threshold, in bytes.  Default 100kb.  See
L<LazyPatricia> for details.

=item --completionLimit <number>

Maximum number of completions to store for a given prefix.  Default 50.

=item --verbose

Print more progress messages.

=item --help | -h | -?

Print a usage message.

=back

=cut

use strict;
use warnings;

use FindBin qw($Bin);
use lib "$Bin/../src/perl5";
use JBlibs;

use Fcntl ":flock";
use File::Spec::Functions;
use Getopt::Long;
use IO::File;
use Pod::Usage;

use JSON 2;


use LazyPatricia;

use Bio::JBrowse::HashStore;
use GenomeDB;

use Data::Dumper;
$Data::Dumper::Indent = 1;

my %trackHash;
my @includedTrackNames;
my @tracksWithNames;

# TODO: change to hash-based storage, pre-generate prefixes

my $outDir = "data";
my $thresh = 100 * 2**10;
my $verbose = 0;
my $incremental;
my $help;
my $max_completions = 50;
GetOptions("dir|out=s" => \$outDir,
           "thresh=i" => \$thresh,
           "completionLimit=i" => \$max_completions,
           "verbose+" => \$verbose,
           "add"      => \$incremental,
           'tracks=s' => \@includedTrackNames,
           "help|h|?" => \$help) or pod2usage();

my %includedTrackNames = map { $_ => 1 }
                         map { split ',', $_ }
                         @includedTrackNames;

pod2usage( -verbose => 2 ) if $help;

unless (-d $outDir) {
    die <<OUTDIR;
Can't find directory "$outDir".
Run this program from a different working directory,
or specify the location of the output directory with
the --dir command line option.
OUTDIR
}

my $gdb = GenomeDB->new( $outDir );
my $nameStore = Bio::JBrowse::HashStore->open( dir => catdir( $outDir, "names" ) );

my @refSeqs  = @{ $gdb->refSeqs   };
my @tracks   = grep { !%includedTrackNames || $includedTrackNames{ $_->{label} } }
               @{ $gdb->trackList || [] };
unless( @tracks ) {
    warn "No tracks defined in configuration, nothing to do.\n";
}

if( $verbose ) {
    print STDERR "Tracks:\n".join('', map "    $_->{label}\n", @tracks );
}


# clear out old data
{'fraction_non_alphanumeric': 0.14568380213385063, 'fraction_numerical': 0.009699321047526674, 'mean_word_length': 2.8448918717375093, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4472543', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1691, 'n_words': 534}