<commit_before>#!/usr/bin/env perl
use 5.012;
use warnings;
use Getopt::Long;

=head1 NAME

C<gen_ab_lines.pl> - Generates lines of text consisting of only as and bs.

=head1 USAGE

Generate a 250mb file, with average line length of 1000 characters

    gen_ab_lines.pl -l 1000 -b 250000000

Generate a 1000 line file, with average line length of 10000 characters

    gen_ab_lines.pl -l 10000 -n 1000

Generate an endless stream of lines with average length of 500 characters

    gen_ab_lines.pl -l 500

=cut

# Average line length
my $avglen = 1000;
# Number of lines emitted
my $n = undef;
# Number of bytes emitted (approximately)
my $b = undef;

GetOptions("length=i" => \$avglen,
           "num=i"    => \$n,
           "bytes=i"  => \$b);

while (1) {
    last if ((defined $n && $n <= 0) || (defined $b && $b <= 0));

    my $i = int(rand() * (2 * $avglen + 2));

    $b -= 1 if defined $b;

    if ($i < $avglen) {
        print "a";
    } elsif ($i < 2*$avglen) {
        print "b";
    } else {
        print "\n";
        $n -= 1 if defined $n;

        last if ((defined $n && $n <= 0) || (defined $b && $b <= 0));
    }
}<commit_msg>Always terminate generated data with newline<commit_after>#!/usr/bin/env perl
use 5.012;
use warnings;
use Getopt::Long;

=head1 NAME

C<gen_ab_lines.pl> - Generates lines of text consisting of only as and bs.

=head1 USAGE

Generate a 250mb file, with average line length of 1000 characters

    gen_ab_lines.pl -l 1000 -b 250000000

Generate a 1000 line file, with average line length of 10000 characters

    gen_ab_lines.pl -l 10000 -n 1000

Generate an endless stream of lines with average length of 500 characters

    gen_ab_lines.pl -l 500

=cut

# Average line length
my $avglen = 1000;
# Number of lines emitted
my $n = undef;
# Number of bytes emitted (approximately)
my $b = undef;

GetOptions("length=i" => \$avglen,
           "num=i"    => \$n,
           "bytes=i"  => \$b);
my $lastnl = 0;
while (1) {
    last if ((defined $n && $n <= 0) || (defined $b && $b <= 0));

    my $i = int(rand() * (2 * $avglen + 2));

    $b -= 1 if defined $b;

    if ($i < $avglen) {
        print "a";
        $lastnl = 0;
    } elsif ($i < 2*$avglen) {
        print "b";
        $lastnl = 0;
    } else {
        print "\n";
        $lastnl = 1;
        $n -= 1 if defined $n;

        last if ((defined $n && $n <= 0) || (defined $b && $b <= 0));
    }
}

# Always add a newline at the end.
print "\n" if (!$lastnl);
{'fraction_non_alphanumeric': 0.14499797488861887, 'fraction_numerical': 0.053462940461725394, 'mean_word_length': 2.4545454545454546, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6699076', 'n_tokens_mistral': 1024, 'n_tokens_neox': 865, 'n_words': 325}