<commit_before>#!/usr/bin/perl -w
#
# Program:  NightlyTest.pl
#
# Synopsis: Perform a series of tests which are designed to be run nightly.
#           This is used to keep track of the status of the LLVM tree, tracking
#           regressions and performance changes.  This generates one web page a
#           day which can be used to access this information.
#
# Syntax:   NightlyTest.pl [OPTIONS] [CVSROOT BUILDDIR WEBDIR]
#   where
# OPTIONS may include one or more of the following:
#  -nocheckout      Do not create, checkout, update, or configure
#                   the source tree.
#  -noremove        Do not remove the BUILDDIR after it has been built.
#  -nofeaturetests  Do not run the feature tests.
#  -noregressiontests Do not run the regression tests.
#  -notest          Do not even attempt to run the test programs. Implies
#                   -norunningtests.
#  -norunningtests  Do not run the Olden benchmark suite with
#                   LARGE_PROBLEM_SIZE enabled.
#  -parallel        Run two parallel jobs with GNU Make.
#  -release         Build an LLVM Release version
#  -pedantic        Enable additional GCC warnings to detect possible errors.
#  -enable-linscan  Enable linearscan tests
#  -disable-codegen Disable LLC and JIT tests in the nightly tester.
#  -verbose         Turn on some debug output
#  -debug           Print information useful only to maintainers of this script.
#  -nice            Checkout/Configure/Build with "nice" to reduce impact 
#                   on busy servers.
#  -gnuplotscript   Next argument specifies gnuplot script to use
#  -templatefile    Next argument specifies template file to use
#
# CVSROOT is the CVS repository from which the tree will be checked out,
#  specified either in the full :method:user@host:/dir syntax, or
#  just /dir if using a local repo.
# BUILDDIR is the directory where sources for this test run will be checked out
#  AND objects for this test run will be built. This directory MUST NOT
#  exist before the script is run; it will be created by the cvs checkout
#  process and erased (unless -noremove is specified; see above.)
# WEBDIR is the directory into which the test results web page will be written,
#  AND in which the "index.html" is assumed to be a symlink to the most recent
#  copy of the results. This directory MUST exist before the script is run.
# LLVMGCCDIR is the directory in which the LLVM GCC Front End is installed
#  to. This is the same as you would have for a normal LLVM build.
#
use POSIX qw(strftime);

my $HOME = $ENV{'HOME'};
my $CVSRootDir = $ENV{'CVSROOT'};
   $CVSRootDir = "/home/vadve/shared/PublicCVS"
     unless $CVSRootDir;
my $BuildDir   = $ENV{'BUILDDIR'};
   $BuildDir   = "$HOME/buildtest"
     unless $BuildDir;
my $WebDir     = $ENV{'WEBDIR'};
   $WebDir     = "$HOME/cvs/testresults-X86"
     unless $WebDir;

# Calculate the date prefix...
@TIME = localtime;
my $DATE = sprintf "%4d-%02d-%02d", $TIME[5]+1900, $TIME[4]+1, $TIME[3];
my $DateString = strftime "%B %d, %Y", localtime;
my $TestStartTime = gmtime;

# Command line argument settings...
my $NOCHECKOUT = 0;
my $NOREMOVE   = 0;
my $NOFEATURES = 0;
my $NOREGRESSIONS = 0;
my $NOTEST     = 0;
my $NORUNNINGTESTS = 0;
my $MAKEOPTS   = "";
my $PROGTESTOPTS = "";
my $VERBOSE  = 0;
my $DEBUG = 0;
my $CONFIGUREARGS = "--enable-jit";
my $NICE = "";

sub ReadFile {
  if (open (FILE, $_[0])) {
    undef $/;
    my $Ret = <FILE>;
    close FILE;
    $/ = '\n';
    return $Ret;
  } else {
    print "Could not open file '$_[0]' for reading!";
    return "";
  }
}

sub WriteFile {  # (filename, contents)
  open (FILE, ">$_[0]") or die "Could not open file '$_[0]' for writing!";
  print FILE $_[1];
  close FILE;
}

sub GetRegex {   # (Regex with ()'s, value)
  $_[1] =~ /$_[0]/m;
  if (defined($1)) {
    return $1;
  }
  return "?";
}

sub AddRecord {
  my ($Val, $Filename) = @_;
  my @Records;
  if (open FILE, "$WebDir/$Filename") {
    @Records = grep !/$DATE/, split "\n", <FILE>;<commit_msg>Make sure GetRegEx returns something gnuplot can deal with (a number, not a ?) so that graphs don't bail if something fails in a test.<commit_after>#!/usr/bin/perl -w
#
# Program:  NightlyTest.pl
#
# Synopsis: Perform a series of tests which are designed to be run nightly.
#           This is used to keep track of the status of the LLVM tree, tracking
#           regressions and performance changes.  This generates one web page a
#           day which can be used to access this information.
#
# Syntax:   NightlyTest.pl [OPTIONS] [CVSROOT BUILDDIR WEBDIR]
#   where
# OPTIONS may include one or more of the following:
#  -nocheckout      Do not create, checkout, update, or configure
#                   the source tree.
#  -noremove        Do not remove the BUILDDIR after it has been built.
#  -nofeaturetests  Do not run the feature tests.
#  -noregressiontests Do not run the regression tests.
#  -notest          Do not even attempt to run the test programs. Implies
#                   -norunningtests.
#  -norunningtests  Do not run the Olden benchmark suite with
#                   LARGE_PROBLEM_SIZE enabled.
#  -parallel        Run two parallel jobs with GNU Make.
#  -release         Build an LLVM Release version
#  -pedantic        Enable additional GCC warnings to detect possible errors.
#  -enable-linscan  Enable linearscan tests
#  -disable-codegen Disable LLC and JIT tests in the nightly tester.
#  -verbose         Turn on some debug output
#  -debug           Print information useful only to maintainers of this script.
#  -nice            Checkout/Configure/Build with "nice" to reduce impact 
#                   on busy servers.
#  -gnuplotscript   Next argument specifies gnuplot script to use
#  -templatefile    Next argument specifies template file to use
#
# CVSROOT is the CVS repository from which the tree will be checked out,
#  specified either in the full :method:user@host:/dir syntax, or
#  just /dir if using a local repo.
# BUILDDIR is the directory where sources for this test run will be checked out
#  AND objects for this test run will be built. This directory MUST NOT
#  exist before the script is run; it will be created by the cvs checkout
#  process and erased (unless -noremove is specified; see above.)
# WEBDIR is the directory into which the test results web page will be written,
#  AND in which the "index.html" is assumed to be a symlink to the most recent
#  copy of the results. This directory MUST exist before the script is run.
# LLVMGCCDIR is the directory in which the LLVM GCC Front End is installed
#  to. This is the same as you would have for a normal LLVM build.
#
use POSIX qw(strftime);

my $HOME = $ENV{'HOME'};
my $CVSRootDir = $ENV{'CVSROOT'};
   $CVSRootDir = "/home/vadve/shared/PublicCVS"
     unless $CVSRootDir;
my $BuildDir   = $ENV{'BUILDDIR'};
   $BuildDir   = "$HOME/buildtest"
     unless $BuildDir;
my $WebDir     = $ENV{'WEBDIR'};
   $WebDir     = "$HOME/cvs/testresults-X86"
     unless $WebDir;

# Calculate the date prefix...
@TIME = localtime;
my $DATE = sprintf "%4d-%02d-%02d", $TIME[5]+1900, $TIME[4]+1, $TIME[3];
my $DateString = strftime "%B %d, %Y", localtime;
my $TestStartTime = gmtime;

# Command line argument settings...
my $NOCHECKOUT = 0;
my $NOREMOVE   = 0;
my $NOFEATURES = 0;
my $NOREGRESSIONS = 0;
my $NOTEST     = 0;
my $NORUNNINGTESTS = 0;
my $MAKEOPTS   = "";
my $PROGTESTOPTS = "";
my $VERBOSE  = 0;
my $DEBUG = 0;
my $CONFIGUREARGS = "--enable-jit";
my $NICE = "";

sub ReadFile {
  if (open (FILE, $_[0])) {
    undef $/;
    my $Ret = <FILE>;
    close FILE;
    $/ = '\n';
    return $Ret;
  } else {
    print "Could not open file '$_[0]' for reading!";
    return "";
  }
}

sub WriteFile {  # (filename, contents)
  open (FILE, ">$_[0]") or die "Could not open file '$_[0]' for writing!";
  print FILE $_[1];
  close FILE;
}

sub GetRegex {   # (Regex with ()'s, value)
  $_[1] =~ /$_[0]/m;
  if (defined($1)) {
    return $1;
  }
  return "0";
}

sub AddRecord {
  my ($Val, $Filename) = @_;
  my @Records;
  if (open FILE, "$WebDir/$Filename") {
    @Records = grep !/$DATE/, split "\n", <FILE>;
{'fraction_non_alphanumeric': 0.1060214259327669, 'fraction_numerical': 0.008003940401428395, 'mean_word_length': 2.91988416988417, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728992', 'n_tokens_mistral': 2666, 'n_tokens_neox': 2592, 'n_words': 1051}