<commit_before>#!/usr/bin/perl

# This script (optionally) runs a coverage test, then generates html reports.
# It requires gcc and gcov to be installed.

use File::Basename;
use File::Find;
use POSIX qw(strftime);

use strict;

my $SRCDIR  = "./"; # with trailing /
my $HTMLDIR = "parrot_coverage";
my $DEBUG   = 1;

if ($ARGV[0] =~ /recompile/) {

    #### clean up remnants of prior biulds
    File::Find::find({wanted => sub {
                          /\.(bb|bba|bbf|da|gcov)$/ &&
                            unlink($File::Find::name)
                        }}, $SRCDIR);

    #### build parrot with coverage support
    system("perl Configure.pl --defaults --define cc=\"gcc -fprofile-arcs -ftest-coverage\"");
    system("make");

    #### Now run the tests
    system("make test");
}

#### And generate the reports.

my @dafiles;
File::Find::find({wanted => sub {
                  /\.da$/ && push @dafiles, $File::Find::name }}, $SRCDIR);

my (%file_line_coverage, %file_branch_coverage, %file_call_coverage);
my (%function_line_coverage, %function_branch_coverage, %function_call_coverage);
my (%totals, %real_filename);

foreach my $da_file (@dafiles) {
    my $dirname   = dirname($da_file) || ".";
    my $filename  = basename($da_file);

    my $cmd = "cd $dirname; gcov -f -b $filename";
    print "Running $cmd..\n" if $DEBUG;
    open (GCOVSUMMARY, "$cmd|") || die "Error invoking '$cmd': $!";
    my $tmp;
    my %generated_files;
    while (<GCOVSUMMARY>) {
        if (/^Creating (.*)\./) {
            my $path = "$dirname/$1"; $path =~ s/\Q$SRCDIR\E//g;
            $generated_files{$path} = $tmp;
            $tmp = '';
        } else {
            $tmp .= $_;
        }
    }
    close(GCOVSUMARY);

    foreach my $gcov_file (keys %generated_files) {
        my $source_file = $gcov_file;
        $source_file =~ s/\.gcov$//g;

        print "Processing $gcov_file ($source_file)\n";

        foreach (split "\n", $generated_files{$gcov_file}) {
            my ($percent, $total_lines, $real_filename) = /\s*([^%]+)% of (\d+) source lines executed in file (.*)/;
            if ($total_lines) {
                my $covered_lines = int(($percent/100) * $total_lines);
                $totals{lines} += $total_lines;
                $totals{covered_lines} += $covered_lines;
                $file_line_coverage{$source_file} = $percent;
                $real_filename{$source_file} = $real_filename;
                next;
            }

            my ($percent, $total_lines, $function) = /\s*([^%]+)% of (\d+) source lines executed in function (.*)/;
            if ($total_lines) {<commit_msg>Fix handling of core.ops, which is used to generate several .c files- don't overwrite the results of one with the other.<commit_after>#!/usr/bin/perl

# This script (optionally) runs a coverage test, then generates html reports.
# It requires gcc and gcov to be installed.

use File::Basename;
use File::Find;
use POSIX qw(strftime);

use strict;

my $SRCDIR  = "./"; # with trailing /
my $HTMLDIR = "parrot_coverage";
my $DEBUG   = 1;

if ($ARGV[0] =~ /recompile/) {

    #### clean up remnants of prior biulds
    File::Find::find({wanted => sub {
                          /\.(bb|bba|bbf|da|gcov)$/ &&
                            unlink($File::Find::name)
                        }}, $SRCDIR);

    #### build parrot with coverage support
    system("perl Configure.pl --defaults --define cc=\"gcc -fprofile-arcs -ftest-coverage\"");
    system("make");

    #### Now run the tests
    system("make test");
}

#### And generate the reports.

my @dafiles;
File::Find::find({wanted => sub {
                  /\.da$/ && push @dafiles, $File::Find::name }}, $SRCDIR);

my (%file_line_coverage, %file_branch_coverage, %file_call_coverage);
my (%function_line_coverage, %function_branch_coverage, %function_call_coverage);
my (%totals, %real_filename);

foreach my $da_file (@dafiles) {
    my $dirname   = dirname($da_file) || ".";
    my $filename  = basename($da_file);

    my $cmd = "cd $dirname; gcov -f -b $filename";
    print "Running $cmd..\n" if $DEBUG;
    open (GCOVSUMMARY, "$cmd|") || die "Error invoking '$cmd': $!";
    my $tmp;
    my %generated_files;
    while (<GCOVSUMMARY>) {
        if (/^Creating (.*)\./) {
            my $path = "$dirname/$1"; $path =~ s/\Q$SRCDIR\E//g;
            $generated_files{$path} = $tmp;
            $tmp = '';
        } else {
            $tmp .= $_;
        }
    }
    close(GCOVSUMARY);

    foreach my $gcov_file (keys %generated_files) {
        my $source_file = $gcov_file;
        $source_file =~ s/\.gcov$//g;

        # avoid collisions where multiple files are generated from the 
        # same back-end file (core.ops, for example)
        if (exists($file_line_coverage{$source_file})) {
            $source_file = "$source_file (from $da_file)";
        }

        print "Processing $gcov_file ($source_file)\n";

        foreach (split "\n", $generated_files{$gcov_file}) {
            my ($percent, $total_lines, $real_filename) = /\s*([^%]+)% of (\d+) source lines executed in file (.*)/;
            if ($total_lines) {
                my $covered_lines = int(($percent/100) * $total_lines);
                $totals{lines} += $total_lines;
                $totals{covered_lines} += $covered_lines;
                $file_line_coverage{$source_file} = $percent;
                $real_filename{$source_file} = $real_filename;
                next;
            }

            my ($percent, $total_lines, $function) = /\s*([^%]+)% of (\d+) source lines executed in function (.*)/;
            if ($total_lines) {
{'fraction_non_alphanumeric': 0.16752991605643866, 'fraction_numerical': 0.0021432398642614752, 'mean_word_length': 2.3674082982561635, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7513665', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1786, 'n_words': 449}