<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

use Getopt::Long;

my $inputfile = "input.txt";
my $outputfile = "output.txt";

GetOptions(
    'i|in|inputfile=s' => \$inputfile,
    'o|out|outputfile=s' => \$outputfile
    );

my $infile; # variable to store the input filehandle

# check if the input file ist a compressed file
if ($inputfile =~ /\.gz|\.bz2/)
{
    require IO::Uncompress::AnyUncompress;
    $infile = new IO::Uncompress::AnyUncompress $inputfile || die "Unable to open input file '$inputfile': $!";
} else {
    open($infile, "<", $inputfile) || die "Unable to open input file '$inputfile': $!";
}

# I want to work on blocks of maximum $max_ids_per_block gis
my $max_ids_per_block = 1000;

# generate a comma seperated list of gis
my $gilist;
my @block = ();
while (<$infile>)
{
    chomp($_);

    push(@block, $_+0);

    if (@block >= $max_ids_per_block || eof($infile))
    {
	my $gilist = join(",", @block);
	print $gilist;

	@block = ();
    }
}
<commit_msg>Added retieval of accessions and add the output to the output file<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

use LWP::Simple;

use Getopt::Long;

my $inputfile = "input.txt";
my $outputfile = "output.txt";

GetOptions(
    'i|in|inputfile=s' => \$inputfile,
    'o|out|outputfile=s' => \$outputfile
    );

my $infile; # variable to store the input filehandle

# check if the input file ist a compressed file
if ($inputfile =~ /\.gz|\.bz2/)
{
    require IO::Uncompress::AnyUncompress;
    $infile = new IO::Uncompress::AnyUncompress $inputfile || die "Unable to open input file '$inputfile': $!";
} else {
    open($infile, "<", $inputfile) || die "Unable to open input file '$inputfile': $!";
}

# I want to work on blocks of maximum $max_ids_per_block gis
my $max_ids_per_block = 1000;

# prepare the output file
my $outfile;
if ($outputfile ne "-")
{
    open($outfile, ">", $outputfile) || die "Unable to open input file '$inputfile': $!";
} else {
    $outfile = \*STDOUT;
}

# generate a comma seperated list of gis
my $gilist;
my @block = ();
while (<$infile>)
{
    chomp($_);

    push(@block, $_+0);

    if (@block >= $max_ids_per_block || eof($infile))
    {
	my $gilist = join(",", @block);

	#assemble the URL
	my $base = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/';
	my $url = $base . "efetch.fcgi?db=nucleotide&id=$gilist&rettype=acc";

	#post the URL
	my @accs = split(/\r*\n/, get($url));

	# check if both lists have the same length
	if (@accs != @block)
	{
	    die "Different length of input and output list";
	}
	for(my $i=0; $i<@block; $i++)
	{
	    printf $outfile "%s\t%s\n", $block[$i], $accs[$i];
	}

	@block = ();
    }
}

close($outfile) || die "Unable to close the output file '$outputfile': $!";

{'fraction_non_alphanumeric': 0.17629952744456562, 'fraction_numerical': 0.004725554343874954, 'mean_word_length': 3.8027923211169283, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5359887', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1007, 'n_words': 290}