<commit_before>#!/usr/bin/env perl

# release_gvf.pl - Create GVF dumps for an ensembl release
#
# see the end of the file for documentation

use strict;
use warnings;

use Data::Dumper;
use Getopt::Long;
use Pod::Usage;
use File::Path qw(make_path);
use FindBin qw($Bin);

my $registry;
my $toplevel_dir;
my $script;
my $test;
my @species;
my @ignore;
my $help;

GetOptions(
    "registry|r=s"      => \$registry,
    "output_dir|o=s"    => \$toplevel_dir,
    "script|s=s"        => \$script, 
    "test|t"            => \$test,
    "species|s=s"       => \@species,
    "ignore|i=s"        => \@ignore,
    "help|h"            => \$help,
) or die pod2usage(1);

pod2usage(1) if $help;

die "--output_dir argument is required, try --help for usage" 
    unless $toplevel_dir;

$script ||= "$Bin/dump_gvf.pl";

my $default_rc = '-q long';

my %to_ignore = map { $_ => 1 } @ignore;

my $species_config = {
    'Homo_sapiens'              => {
        individuals     => {
            Watson  => 'ENSEMBL:Watson',
            Venter  => 'ENSEMBL:Venter',
        },
        populations     => {
            '1000GENOMES:low_coverage:CEU'      => '1000 genomes - Low coverage - CEU',
            '1000GENOMES:low_coverage:CHBJPT'   => '1000 genomes - Low coverage - CHB+JPT',
            '1000GENOMES:low_coverage:YRI'      => '1000 genomes - Low coverage - YRI',
            '1000GENOMES:trio:CEU'              => '1000 genomes - High coverage - Trios - CEU', 
            '1000GENOMES:trio:YRI'              => '1000 genomes - High coverage - Trios - YRI',
            '1000GENOMES:exon:CEU'              => '1000 genomes - High coverage exons - CEU',
            '1000GENOMES:exon:CHB'              => '1000 genomes - High coverage exons - CHB',
            '1000GENOMES:exon:CHD'              => '1000 genomes - High coverage exons - CHD',
            '1000GENOMES:exon:JPT'              => '1000 genomes - High coverage exons - JPT',
            '1000GENOMES:exon:LWK'              => '1000 genomes - High coverage exons - LWK',
            '1000GENOMES:exon:TSI'              => '1000 genomes - High coverage exons - TSI',
            '1000GENOMES:exon:YRI'              => '1000 genomes - High coverage exons - YRI',
            'CSHL-HAPMAP:HapMap-CEU'            => '',
            'CSHL-HAPMAP:HapMap-HCB'            => '',
            'CSHL-HAPMAP:HapMap-JPT'            => '',
            'CSHL-HAPMAP:HapMap-YRI'            => '',
            'CSHL-HAPMAP:HAPMAP-ASW'            => '',
            'CSHL-HAPMAP:HAPMAP-CHB'            => '',
            'CSHL-HAPMAP:HAPMAP-CHD'            => '',
            'CSHL-HAPMAP:HAPMAP-GIH'            => '',
            'CSHL-HAPMAP:HAPMAP-LWK'            => '',
            'CSHL-HAPMAP:HAPMAP-MEX'            => '',
            'CSHL-HAPMAP:HAPMAP-MKK'            => '',
            'CSHL-HAPMAP:HAPMAP-TSI'            => '',
        },
        rc              => '-q long -R"select[mem>15000] rusage[mem=15000]" -M15000000',
        somatic         => 1,
    },
    'Mus_musculus'	            => {},
    'Danio_rerio'	            => {},
    'Felis_catus'	            => {},
    'Bos_taurus'	            => {},
    'Canis_familiaris'	        => {},
    'Equus_caballus'	        => {},
    'Taeniopygia_guttata'	    => {},
    'Tetraodon_nigroviridis'    => {},
    'Monodelphis_domestica'	    => {},
    'Sus_scrofa'	            => {},
    'Ornithorhynchus_anatinus'	=> {},
    'Pan_troglodytes'	        => {},
    'Pongo_pygmaeus'	        => {},
    'Rattus_norvegicus'	        => {},
    'Gallus_gallus'	            => {},
    'Drosophila_melanogaster'	=> {},
    'Saccharomyces_cerevisiae'	=> {},
};

unless (-e $toplevel_dir || $test) {
    make_path($toplevel_dir) or die "Failed to create toplevel output directory '$toplevel_dir': $!";
}

my $normal_opts = "--species %s --registry %s --compress --chunk_size 100";

my @cmds;

@species = keys %$species_config unless @species;

for my $species_name (@species) {

    next if $to_ignore{$species_name};
    
    my $config = $species_config->{$species_name};
    my $rc = $config->{rc} || $default_rc;

    my $dir = $toplevel_dir.'/'.$species_name;
    
    unless (-e $dir || $test) {<commit_msg>make each species directory lower case<commit_after>#!/usr/bin/env perl

# release_gvf.pl - Create GVF dumps for an ensembl release
#
# see the end of the file for documentation

use strict;
use warnings;

use Data::Dumper;
use Getopt::Long;
use Pod::Usage;
use File::Path qw(make_path);
use FindBin qw($Bin);

my $registry;
my $toplevel_dir;
my $script;
my $test;
my @species;
my @ignore;
my $help;

GetOptions(
    "registry|r=s"      => \$registry,
    "output_dir|o=s"    => \$toplevel_dir,
    "script|s=s"        => \$script, 
    "test|t"            => \$test,
    "species|s=s"       => \@species,
    "ignore|i=s"        => \@ignore,
    "help|h"            => \$help,
) or die pod2usage(1);

pod2usage(1) if $help;

die "--output_dir argument is required, try --help for usage" 
    unless $toplevel_dir;

$script ||= "$Bin/dump_gvf.pl";

my $default_rc = '-q long';

my %to_ignore = map { $_ => 1 } @ignore;

my $species_config = {
    'Homo_sapiens'              => {
        individuals     => {
            Watson  => 'ENSEMBL:Watson',
            Venter  => 'ENSEMBL:Venter',
        },
        populations     => {
            '1000GENOMES:low_coverage:CEU'      => '1000 genomes - Low coverage - CEU',
            '1000GENOMES:low_coverage:CHBJPT'   => '1000 genomes - Low coverage - CHB+JPT',
            '1000GENOMES:low_coverage:YRI'      => '1000 genomes - Low coverage - YRI',
            '1000GENOMES:trio:CEU'              => '1000 genomes - High coverage - Trios - CEU', 
            '1000GENOMES:trio:YRI'              => '1000 genomes - High coverage - Trios - YRI',
            '1000GENOMES:exon:CEU'              => '1000 genomes - High coverage exons - CEU',
            '1000GENOMES:exon:CHB'              => '1000 genomes - High coverage exons - CHB',
            '1000GENOMES:exon:CHD'              => '1000 genomes - High coverage exons - CHD',
            '1000GENOMES:exon:JPT'              => '1000 genomes - High coverage exons - JPT',
            '1000GENOMES:exon:LWK'              => '1000 genomes - High coverage exons - LWK',
            '1000GENOMES:exon:TSI'              => '1000 genomes - High coverage exons - TSI',
            '1000GENOMES:exon:YRI'              => '1000 genomes - High coverage exons - YRI',
            'CSHL-HAPMAP:HapMap-CEU'            => '',
            'CSHL-HAPMAP:HapMap-HCB'            => '',
            'CSHL-HAPMAP:HapMap-JPT'            => '',
            'CSHL-HAPMAP:HapMap-YRI'            => '',
            'CSHL-HAPMAP:HAPMAP-ASW'            => '',
            'CSHL-HAPMAP:HAPMAP-CHB'            => '',
            'CSHL-HAPMAP:HAPMAP-CHD'            => '',
            'CSHL-HAPMAP:HAPMAP-GIH'            => '',
            'CSHL-HAPMAP:HAPMAP-LWK'            => '',
            'CSHL-HAPMAP:HAPMAP-MEX'            => '',
            'CSHL-HAPMAP:HAPMAP-MKK'            => '',
            'CSHL-HAPMAP:HAPMAP-TSI'            => '',
        },
        rc              => '-q long -R"select[mem>15000] rusage[mem=15000]" -M15000000',
        somatic         => 1,
    },
    'Mus_musculus'	            => {},
    'Danio_rerio'	            => {},
    'Felis_catus'	            => {},
    'Bos_taurus'	            => {},
    'Canis_familiaris'	        => {},
    'Equus_caballus'	        => {},
    'Taeniopygia_guttata'	    => {},
    'Tetraodon_nigroviridis'    => {},
    'Monodelphis_domestica'	    => {},
    'Sus_scrofa'	            => {},
    'Ornithorhynchus_anatinus'	=> {},
    'Pan_troglodytes'	        => {},
    'Pongo_pygmaeus'	        => {},
    'Rattus_norvegicus'	        => {},
    'Gallus_gallus'	            => {},
    'Drosophila_melanogaster'	=> {},
    'Saccharomyces_cerevisiae'	=> {},
};

unless (-e $toplevel_dir || $test) {
    make_path($toplevel_dir) or die "Failed to create toplevel output directory '$toplevel_dir': $!";
}

my $normal_opts = "--species %s --registry %s --compress --chunk_size 100";

my @cmds;

@species = keys %$species_config unless @species;

for my $species_name (@species) {

    next if $to_ignore{$species_name};
    
    my $config = $species_config->{$species_name};
    my $rc = $config->{rc} || $default_rc;

    # the webteam want the toplevel species directory to be lower case
    my $dir = $toplevel_dir.'/'.lc($species_name);
    
    unless (-e $dir || $test) {
{'fraction_non_alphanumeric': 0.15762910798122065, 'fraction_numerical': 0.028873239436619718, 'mean_word_length': 1.8196558570483123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 121, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '791295', 'n_tokens_mistral': 3321, 'n_tokens_neox': 2955, 'n_words': 554}