<commit_before>#!/usr/bin/perl -W 
# script for preprocessing language data prior to tokenization
# Start by <NAME>, after noticing systematic preprocessing errors
# in some of the English Europarl data.

use strict;
use Getopt::Std;

binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");

sub usage
{
  print "Script for preprocessing of raw language data prior to tokenization\n";
  print "Usage: $0 -l <language tag>\n";
}

my %args;
getopt('l=s h',\%args);
usage() && exit(0) if $args{'h'};

if ($args{'l'} eq "en")
  {
      while (<>)
      {
	  s/([[:alpha:]]\') s\b/$1s/g;
	  print;
      }
      
  }
elsif ($args{'l'} eq "fr")
  {
    while (<>)
      {
	s/\b([[:alpha:]]\')\s+(?=[[:alpha:]])/$1/g;
	print;
      }
  }
else
  {
    
    print while <>;
  }
<commit_msg>Added -b switch to pretokenizer to allow disabling of buffering.<commit_after>#!/usr/bin/perl -W
# script for preprocessing language data prior to tokenization
# Start by <NAME>, after noticing systematic preprocessing errors
# in some of the English Europarl data.

use strict;
use Getopt::Std;

binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");

sub usage
{
  print "Script for preprocessing of raw language data prior to tokenization\n";
  print "Usage: $0 -l <language tag> [-b]\n";
  print "       -b: no buffering\n";
}

my %args;
getopt('l=s h b',\%args);
usage() && exit(0) if $args{'h'};
$|++ if $args{'b'};
if ($args{'l'} eq "en")
  {
    while (<>)
      {
	s/([[:alpha:]]\') s\b/$1s/g;
	print;
      }
  }
elsif ($args{'l'} eq "fr")
  {
    while (<>)
      {
	s/\b([[:alpha:]]\')\s+(?=[[:alpha:]])/$1/g;
	print;
      }
  }
else
  {
    print while <>;
  }

{'fraction_non_alphanumeric': 0.2034001214329083, 'fraction_numerical': 0.007285974499089253, 'mean_word_length': 3.0491400491400493, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7303362', 'n_tokens_mistral': 648, 'n_tokens_neox': 632, 'n_words': 173}