<commit_before>######
my $script_version="0.1.0";

use Getopt::Std;
use strict;
use warnings;

## either fetch CRAN or Bioconductor, defaults to pull.
## push pushes a local mirror to another directory.

my $cran="cran.r-project.org::CRAN";       ## that's the url to the CRAN
my $bioc="master.bioconductor.org::3.0";  ## that's the url from which we fetch bioconductor.
my $localcran="/Volumes/jodata/mirror/CRAN";     ## that's where we store locally the CRAN.
my $localbioc="/Volumes/jodata/mirror/bioconductor/bioc_3.0";     ## that's where we store locally the bioconductor.

my $localurl;
my $remoteurl;

my %option=();
# remote repo, local repo.
getopts("l:r:s:h",\%option);

my $switch = "bioc";
## define which files/folders we would like to exclude.
##my $excludes = "--exclude '**windows*' --exclude '**macosx/' --exclude '*data/experiment*'";
my $excludes = "--exclude '**windows*' --exclude '**macosx/' --exclude '**linux*' --exclude 'CRAN/bin*' --exclude 'bioc/bin*' --exclude 'bioc/citations*' --exclude 'bioc/html*' --exclude 'bioc/install*' --exclude 'bioc/licenses*' --exclude 'bioc/manuals*' --exclude 'bioc/news*' --exclude 'bioc/readmes*' --exclude 'bioc/vignettes*' --exclude 'contrib/Archive*'";


if( $option{ h } ){
## print help... obviously...
  print( "\nmirrror.pl version ".$script_version.".\n" );
  print( "Mirrors the CRAN or Bioconductor repository.\n");
  print( "usage: mirrror.pl [-hlrs]\n" );
  print( "Parameters:\n" );
  print( "h : print this help.\n" );
  print( "l : path to local copy of the repository. If not provided the default defined within this script will be used.\n" );
  print( "r : url of the remote repository. If not provided the default defined within this script will be used.\n" );
  print( "s : switch to define whether Bioconductor (the default) or CRAN should be mirrored. If defined has to be one of 'bioc' or 'CRAN'.\n\n" );
  exit 0;
}

if( defined( $option{ s } ) ){
  if( $option{ s } eq "CRAN" ){
    $switch = "CRAN";
  }
  elsif( $option{ s } eq "bioc" ){
    $switch = "bioc";
  }
  else{
    die( "parameter s has to be either 'bioc' or 'CRAN'!" );
  }
}<commit_msg>Updates to the mirror function<commit_after>######
my $script_version="0.1.0";

use Getopt::Std;
use strict;
use warnings;

## either fetch CRAN or Bioconductor, defaults to pull.
## push pushes a local mirror to another directory.

my $cran="cran.r-project.org::CRAN";       ## that's the url to the CRAN
my $bioc="master.bioconductor.org::3.0";  ## that's the url from which we fetch bioconductor.
my $localcran="/Volumes/jodata/mirror/CRAN";     ## that's where we store locally the CRAN.
my $localbioc="/Volumes/jodata/mirror/bioconductor/bioc_3.0";     ## that's where we store locally the bioconductor.

my $localurl;
my $remoteurl;

my %option=();
# remote repo, local repo.
getopts("l:r:s:h",\%option);

my $switch = "bioc";
## define which files/folders we would like to exclude.
##my $excludes = "--exclude '**windows*' --exclude '**macosx/' --exclude '*data/experiment*'";
my $excludes = "--exclude '**windows*' --exclude '**macosx/' --exclude '**linux*' --exclude 'bin/*' --exclude 'bioc/bin*' --exclude 'bioc/citations*' --exclude 'bioc/html*' --exclude 'bioc/install*' --exclude 'bioc/licenses*' --exclude 'bioc/manuals*' --exclude 'bioc/news*' --exclude 'bioc/readmes*' --exclude 'bioc/vignettes*' --exclude 'contrib/Archive*'";


if( $option{ h } ){
## print help... obviously...
  print( "\nmirrror.pl version ".$script_version.".\n" );
  print( "Mirrors the CRAN or Bioconductor repository.\n");
  print( "usage: mirrror.pl [-hlrs]\n" );
  print( "Parameters:\n" );
  print( "h : print this help.\n" );
  print( "l : path to local copy of the repository. If not provided the default defined within this script will be used.\n" );
  print( "r : url of the remote repository. If not provided the default defined within this script will be used.\n" );
  print( "s : switch to define whether Bioconductor (the default) or CRAN should be mirrored. If defined has to be one of 'bioc' or 'CRAN'.\n\n" );
  exit 0;
}

if( defined( $option{ s } ) ){
  if( $option{ s } eq "CRAN" ){
    $switch = "CRAN";
  }
  elsif( $option{ s } eq "bioc" ){
    $switch = "bioc";
  }
  else{
    die( "parameter s has to be either 'bioc' or 'CRAN'!" );
  }
}
{'fraction_non_alphanumeric': 0.17758098345374038, 'fraction_numerical': 0.0037287345607084597, 'mean_word_length': 3.700985761226725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7022756', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1483, 'n_words': 492}