<commit_before>#Utility to report the number of chros in bigWig and bigBed files

use strict;
use warnings;
use List::MoreUtils qw(uniq);
use Getopt::Long;

my $ifile="/nfs/1000g-work/ihec/drop/bp-raw-data/blueprint/results.index";
my $base_dir="/nfs/1000g-work/ihec/drop/";
my $bin_dir="/nfs/1000g-work/G1K/work/davidr/pipeline-deps/bin/";
my $help;

my $usage = "
  [--help]          this menu
   --index          .index file containing the paths of the files to be analyzed,
   --base_dir       base_dir used to construct the paths
   --bin_dir        dir containing the binaries 

   [USAGE] perl report_number_chros.pl --index /path/to/results.index --base_dir /nfs/1000g-work/ihec/drop/ --bin_dir /nfs/1000g-work/G1K/work/davidr/pipeline-deps/bin/
";

&GetOptions(
    'index=s'              => \$ifile,
    'base_dir=s'           => \$base_dir,
    'bin_dir=s'            => \$bin_dir
);

if ($help) {
    print $usage;
    exit 0;
}

#NOTE; bigWigInfo and bigBedInfo need to be in $PATH
use strict;
use warnings;
use List::MoreUtils qw(uniq);

# index file
my $ifile=$ARGV[0];
die("[ERROR] I need and *.index file\n") if !$ifile;

#array containing the FILE_TYPEs to be analized
my @valid=qw(BS_METH_CALL_CNAG BS_METH_SD_CNAG CS_FILTERED_BW CS_WIGGLER DS_FILTERED_BW DS_WIGGLER RNA_SIGNAL_CRG BS_HYPER_METH_BB_CNAG BS_HYPO_METH_BB_CNAG CS_MACS_TRACK_BB CS_MACS_TRACK_BB_WP10 CS_MACS_TRACK_BROAD_BB CS_MACS_TRACK_BROAD_BB_WP10 DNASE_TRACK_BB_NCMLS DS_HOTSPOT_TRACK_BB);

my %valid = map { $_ => 1 } @valid;


open FILE,"<$ifile" or die("Cant open $ifile:$!\n");
my $header=<FILE>;
close FILE;
my @fields=split/\t/,$header;

#get indexes for useful columns
my $file_ix = List::MoreUtils::first_index {$_ eq 'FILE'} @fields;
my $file_type_ix = List::MoreUtils::first_index {$_ eq 'FILE_TYPE'} @fields;

#header
print "#file\tchrNo\n";

my $first=0;
open FH,"<$ifile" or die("Cant open $ifile:$!\n");
while(<FH>) {
    chomp;
    if ($first==0) {
	$first=1;
	next;
    }
    my $line=$_;
    my @fields=split/\t/,$line;
    #check if this FILE_TYPE is declared in @valid
    next if !(exists($valid{$fields[$file_type_ix]}));
    my $filepath=$fields[$file_ix];
    my $cmd;
    #check if file is .bw or .bb
    if ($filepath=~/\.bw$/) {
	#create command
	$cmd="$bin_dir/bigWigInfo $base_dir/bp-raw-data/$filepath";
    } elsif ($filepath=~/\.bb$/) {
	$cmd="$bin_dir/bigBedInfo $base_dir/bp-raw-data/$filepath";
	$cmd='bigWigInfo $IHECD/bp-raw-data/'.$filepath;
    } elsif ($filepath=~/\.bb$/) {
	$cmd='bigBedInfo $IHECD/bp-raw-data/'.$filepath;
    }
    #execute command
    my $res=`$cmd`;
    #get chromosome count from $res
    my $chr_count=$1 if $res=~/chromCount: (\d+)/;
    print $filepath,"\t",$chr_count,"\n";
}
close FH;
<commit_msg>modifications in report_number_chros.pl<commit_after>#Utility to report the number of chros in bigWig and bigBed files

use strict;
use warnings;
use List::MoreUtils qw(uniq);
use Getopt::Long;

my $ifile="/nfs/1000g-work/ihec/drop/bp-raw-data/blueprint/results.index";
my $base_dir="/nfs/1000g-work/ihec/drop/";
my $bin_dir="/nfs/1000g-work/G1K/work/davidr/pipeline-deps/bin/";
my $help;

my $usage = "
  [--help]          this menu
   --index          .index file containing the paths of the files to be analyzed,
   --base_dir       base_dir used to construct the paths
   --bin_dir        dir containing the binaries 

   [USAGE] perl report_number_chros.pl --index /path/to/results.index --base_dir /nfs/1000g-work/ihec/drop/ --bin_dir /nfs/1000g-work/G1K/work/davidr/pipeline-deps/bin/
";

&GetOptions(
    'index=s'              => \$ifile,
    'base_dir=s'           => \$base_dir,
    'bin_dir=s'            => \$bin_dir
);

if ($help) {
    print $usage;
    exit 0;
}

#array containing the FILE_TYPEs to be analized
my @valid_files=qw(CHIP_MACS2_BROAD_BB CHIP_WIGGLER CHIP_MACS2_BB);

#array containing chros to be considered for a file to be valid
my @valid_chros= qw(chr1 chr2 chr3 chr4 chr5 chr6 chr7 chr8 chr9 chr10 chr11 chr12 chr13 chr14 chr15 chr16 chr17 chr18 chr19 chr20 chr21 chr22 chrX chrY chrM);

my %valid = map { $_ => 1 } @valid_files;


open FILE,"<$ifile" or die("Cant open $ifile:$!\n");
my $header=<FILE>;
close FILE;
my @fields=split/\t/,$header;

#get indexes for useful columns
my $file_ix = List::MoreUtils::first_index {$_ eq 'FILE'} @fields;
my $file_type_ix = List::MoreUtils::first_index {$_ eq 'FILE_TYPE'} @fields;

#header
print "#file\tchrNo\tvalid\n";

my $first=0;
open FH,"<$ifile" or die("Cant open $ifile:$!\n");
while(<FH>) {
    chomp;
    if ($first==0) {
	$first=1;
	next;
    }
    my $line=$_;
    my @fields=split/\t/,$line;
    #check if this FILE_TYPE is declared in @valid
    next if !(exists($valid{$fields[$file_type_ix]}));
    my $filepath=$fields[$file_ix];
    my $cmd;
    #check if file is .bw or .bb
    if ($filepath=~/\.bw$/) {
	#create command
	$cmd="$bin_dir/bigWigInfo -chroms $base_dir/bp-raw-data/$filepath";
    } elsif ($filepath=~/\.bb$/) {
	$cmd="$bin_dir/bigBedInfo -chroms $base_dir/bp-raw-data/$filepath";
    }
    #execute command
    my $res=`$cmd`;
    #parse output
    my $valid=1;
    my @lines=split/\s/,$res;
    my %chros= map { $_ => 1 } (grep {/chr/} @lines);
    #check if this file has all the chros in @valid_chros
    foreach my $chr (@valid_chros) {
	if (!exists($chros{$chr})) {
	    $valid=0;
	}
    }
    my $number=scalar(keys(%chros));
    print "$filepath\t$number\t$valid\n";
}
close FH;

{'fraction_non_alphanumeric': 0.15993389643775247, 'fraction_numerical': 0.01836210062431142, 'mean_word_length': 3.4942244224422443, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7546829', 'n_tokens_mistral': 2247, 'n_tokens_neox': 2170, 'n_words': 503}