<commit_before>use strict;
use warnings;
use Test::More tests => 84;
use HTTP::Headers;
use HTTP::Status qw(:constants);
use IO::Capture::Stderr;
use JSON;
use XML::XPath;
use English qw(-no_match_vars);
use lib qw(t/headers//lib t/lib);
use t::request;

use t::model::response;
use t::view::response;

my $runner = sub {
  my ($headers_ref, $content_ref, $config) = @_;

  no warnings qw(redefine once);
  local *ClearPress::util::data_path = sub { return 't/headers/data'; };

  my $response = t::request->new($config);

  my ($header_str, $content) = $response =~ m{^(.*?\n)\n(.*)$}smix;
  my $headers = HTTP::Headers->new();

  for my $line (split /\n/smx, $header_str) {
    my ($k, $v) = split m{\s*:\s*}smx, $line, 2;
    $headers->header($k, $v);
  }

  ${$headers_ref} = $headers;
  ${$content_ref} = $content;

  return 1;
};

{
  my $sets = [
	      [ '',     'text/html',        sub { my $arg=shift; return $arg;                                                      } ], # plain # <p class="error">
	      [ '.js',  'application/json', sub { my $arg=shift; return JSON->new->decode($arg)->{error};                          } ], # json
	      [ '.csv', 'application/csv',  sub { my $arg=shift; return [split /[\r\n]+/smix, $arg]->[0];                          } ], # csv
	      [ '.xml', 'text/xml',         sub { my $arg=shift; return XML::XPath->new(content=>$arg)->find('/error')->as_string; } ], # xml
	     ];

  for my $set (@{$sets}) {
    my ($extension, $content_type, $extraction) = @{$set};
    my $tests = [<commit_msg>corrected xml xpath finder; nb this still works on well-formed html docs but is not returning vanilla xml content<commit_after>use strict;
use warnings;
use Test::More tests => 84;
use HTTP::Headers;
use HTTP::Status qw(:constants);
use IO::Capture::Stderr;
use JSON;
use XML::XPath;
use English qw(-no_match_vars);
use lib qw(t/headers//lib t/lib);
use t::request;

use t::model::response;
use t::view::response;

my $runner = sub {
  my ($headers_ref, $content_ref, $config) = @_;

  no warnings qw(redefine once);
  local *ClearPress::util::data_path = sub { return 't/headers/data'; };

  my $response = t::request->new($config);

  my ($header_str, $content) = $response =~ m{^(.*?\n)\n(.*)$}smix;
  my $headers = HTTP::Headers->new();

  for my $line (split /\n/smx, $header_str) {
    my ($k, $v) = split m{\s*:\s*}smx, $line, 2;
    $headers->header($k, $v);
  }

  ${$headers_ref} = $headers;
  ${$content_ref} = $content;

  return 1;
};

{
  my $sets = [
	      [ '',     'text/html',        sub { my $arg=shift; return $arg;                                                      } ], # plain # <p class="error">
	      [ '.js',  'application/json', sub { my $arg=shift; return JSON->new->decode($arg)->{error};                          } ], # json
	      [ '.csv', 'application/csv',  sub { my $arg=shift; return [split /[\r\n]+/smix, $arg]->[0];                          } ], # csv
	      [ '.xml', 'text/xml',         sub { my $arg=shift; return XML::XPath->new(xml=>$arg)->find('/error'); } ], # xml
	     ];

  for my $set (@{$sets}) {
    my ($extension, $content_type, $extraction) = @{$set};
    my $tests = [
{'fraction_non_alphanumeric': 0.20889870621647208, 'fraction_numerical': 0.003155569580309246, 'mean_word_length': 2.6647398843930636, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6160605', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1092, 'n_words': 274}