<commit_before>#!/usr/bin/env nqp
# Copyright (C) 2014, The Perl Foundation


sub MAIN(*@ARGS) {
    my $program := @ARGS.shift;
    my $backend := @ARGS.shift;
    say("# This file automatically generated by $program\n");
    if @ARGS[0] eq '-f' && nqp::elems(@ARGS) >= 2 {
        nqp::shift(@ARGS);
        my $file := nqp::shift(@ARGS);
        my $fh   := open($file, :r);
        while nqp::readlinefh($fh) -> $line {
            if $line ~~ /\S/ {
                $line := subst($line, /\s+/, '', :global);
                nqp::push(@ARGS, $line);
            }
        }
        close($fh);
    }
    my $stderr := nqp::getstderr();
    for @ARGS -> $file {
        say("#line 1 SETTING::$file");
        my $fh := open($file, :r);
        my int $in_cond := 0;
        my int $in_omit := 0;
        my int $line    := 1;
        while nqp::readlinefh($fh) -> $_ {
            if my $x := $_ ~~ / ^ '#?if' \s+ ('!')? \s* (\w+) \s* $ / {
                nqp::die("Nested conditionals not supported") if $in_cond;
                $in_cond := 1;
                $in_omit := $x[0] && $x[1] eq $backend || !$x[0] && $x[1] ne $backend;
                print("\n");
            } elsif $_ ~~ /^ '#?endif' / {
                unless $in_cond {
                    nqp::sayfh($stderr,
                        "#?endif without matching #?if in file $file, line $line"
                    );
                }
                $in_cond := 0;
                $in_omit := 0;
                print("\n");
            } elsif $in_omit {
                print("\n");
            } else {
                print($_) unless nqp::eqat($_,"# vim:",0);
            }
            $line++;
        }
        close($fh);
    }
    say("\n# vim: set ft=perl6 nomodifiable :");
}


# vim: set ft=perl6 :
<commit_msg>Update gen-cat to use NQP high level I/O handles.<commit_after>#!/usr/bin/env nqp
# Copyright (C) 2014, The Perl Foundation


sub MAIN(*@ARGS) {
    my $program := @ARGS.shift;
    my $backend := @ARGS.shift;
    say("# This file automatically generated by $program\n");
    if @ARGS[0] eq '-f' && nqp::elems(@ARGS) >= 2 {
        nqp::shift(@ARGS);
        my $file := nqp::shift(@ARGS);
        my $fh   := open($file, :r, :!chomp);
        while $fh.get -> $line {
            if $line ~~ /\S/ {
                $line := subst($line, /\s+/, '', :global);
                nqp::push(@ARGS, $line);
            }
        }
        close($fh);
    }
    my $stderr := nqp::getstderr();
    for @ARGS -> $file {
        say("#line 1 SETTING::$file");
        my $fh := open($file, :r, :!chomp);
        my int $in_cond := 0;
        my int $in_omit := 0;
        my int $line    := 1;
        while $fh.get -> $_ {
            if my $x := $_ ~~ / ^ '#?if' \s+ ('!')? \s* (\w+) \s* $ / {
                nqp::die("Nested conditionals not supported") if $in_cond;
                $in_cond := 1;
                $in_omit := $x[0] && $x[1] eq $backend || !$x[0] && $x[1] ne $backend;
                print("\n");
            } elsif $_ ~~ /^ '#?endif' / {
                unless $in_cond {
                    stderr().say(
                        "#?endif without matching #?if in file $file, line $line"
                    );
                }
                $in_cond := 0;
                $in_omit := 0;
                print("\n");
            } elsif $in_omit {
                print("\n");
            } else {
                print($_) unless nqp::eqat($_,"# vim:",0);
            }
            $line++;
        }
        close($fh);
    }
    say("\n# vim: set ft=perl6 nomodifiable :");
}


# vim: set ft=perl6 :

{'fraction_non_alphanumeric': 0.19313779745434423, 'fraction_numerical': 0.011068068622025456, 'mean_word_length': 1.3612018288700196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6476715', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1183, 'n_words': 290}