<commit_before>#!/usr/bin/env perl
use warnings;
use strict;
use Term::ANSIColor;
use IO::Handle;
use IPC::Open2;

my $usage="runtests.pl SPEC PROGRAM\nSet ANSI_COLORS_DISABLED=1 if you redirect to a file.\nSet PATT='...' to restrict tests to sections matching a regex.\n";

my $SPEC = shift @ARGV;
my @PROG = @ARGV;
my $PATT=$ENV{'PATT'};

if (!(@PROG && defined $SPEC)) {
  print STDERR $usage;
  exit 1;
}

my $passed = 0;
my $failed = 0;
my $skipped = 0;

# Markdown implementations vary on insignificant whitespace.
# Some leave blanks between block elements, others don't.
# This function tries to normalize the output so it can be
# compared with our test.  tidy takes two arguments: the
# string containing the actual output, and a pathname of the
# file to which the tidied output is to be saved.
sub tidy
{
  my $inpre = 0;
  my $out = "";
  my $outfh;
  open($outfh, '>', \$out);
  for (split /^/, $_[0]) {
    if (/<pre/) {
      $inpre = 1;
    } elsif (/<\/pre/) {
      $inpre = 0;
    }
    # remove \r to allow mixing linux/windows newlines
	s/\r//;
    if ($inpre) {
      print $outfh $_;
    } else {
      # remove leading spaces
      s/^  *//;
      # remove trailing spaces
      s/  *$//;
      # collapse consecutive spaces
      s/  */ /;
      # collapse space before /> in tag
      s/  *\/>/\/>/;
      # skip blank line
      if (/^$/) {
        next;
      }
      print $outfh $_;
    }
  }
  close $outfh;
  return $out;
}

sub dotest
{
  my $markdown = $_[0];
  my $html = $_[1];
  my $testname = $_[2];
  my $actual = "";<commit_msg>Normalizing newlines around <li> in runtests.pl. This now matches the behavior of unit tests.<commit_after>#!/usr/bin/env perl
use warnings;
use strict;
use Term::ANSIColor;
use IO::Handle;
use IPC::Open2;

my $usage="runtests.pl SPEC PROGRAM\nSet ANSI_COLORS_DISABLED=1 if you redirect to a file.\nSet PATT='...' to restrict tests to sections matching a regex.\n";

my $SPEC = shift @ARGV;
my @PROG = @ARGV;
my $PATT=$ENV{'PATT'};

if (!(@PROG && defined $SPEC)) {
  print STDERR $usage;
  exit 1;
}

my $passed = 0;
my $failed = 0;
my $skipped = 0;

# Markdown implementations vary on insignificant whitespace.
# Some leave blanks between block elements, others don't.
# This function tries to normalize the output so it can be
# compared with our test.  tidy takes two arguments: the
# string containing the actual output, and a pathname of the
# file to which the tidied output is to be saved.
sub tidy
{
  my $inpre = 0;
  my $out = "";
  my $outfh;
  open($outfh, '>', \$out);
  for (split /^/, $_[0]) {
    if (/<pre/) {
      $inpre = 1;
    } elsif (/<\/pre/) {
      $inpre = 0;
    }
    # remove \r to allow mixing linux/windows newlines
	s/\r//;
    if ($inpre) {
      print $outfh $_;
    } else {
      # remove leading spaces
      s/^  *//;
      # remove trailing spaces
      s/  *$//;
      # collapse consecutive spaces
      s/  */ /;
      # collapse space before /> in tag
      s/  *\/>/\/>/;
	  # add a newline after <li> tag
	  s/<li>(.+)/<li>\n$1/;
      # skip blank line
      if (/^$/) {
        next;
      }
      print $outfh $_;
    }
  }
  close $outfh;
  return $out;
}

sub dotest
{
  my $markdown = $_[0];
  my $html = $_[1];
  my $testname = $_[2];
  my $actual = "";
{'fraction_non_alphanumeric': 0.15257857796765334, 'fraction_numerical': 0.00823924321025328, 'mean_word_length': 2.421711899791232, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2164448', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1172, 'n_words': 400}