<commit_before>#
# Extract named character references from HTML Living Standard
#
use Mojo::Base -strict;
use Mojo::UserAgent;

my $res = Mojo::UserAgent->new->get('https://html.spec.whatwg.org')->result;
for my $row ($res->dom('#named-character-references-table tbody > tr')->each) {
  my $entity     = $row->at('td > code')->text;
  my $codepoints = $row->children('td')->[1]->text;
  say "$entity $codepoints";
}

1;<commit_msg>Entities need to be trimmed now<commit_after>#
# Extract named character references from HTML Living Standard
#
use Mojo::Base -strict;

use Mojo::UserAgent;
use Mojo::Util 'trim';

my $res = Mojo::UserAgent->new->get('https://html.spec.whatwg.org')->result;
for my $row ($res->dom('#named-character-references-table tbody > tr')->each) {
  my $entity     = trim $row->at('td > code')->text;
  my $codepoints = trim $row->children('td')->[1]->text;
  say "$entity $codepoints";
}

1;
{'fraction_non_alphanumeric': 0.19803063457330417, 'fraction_numerical': 0.00437636761487965, 'mean_word_length': 4.791139240506329, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5653225', 'n_tokens_mistral': 313, 'n_tokens_neox': 315, 'n_words': 77}