<commit_before><commit_msg>Pretty colourful diffs when comparing similar messages.  Hopefully more readable.<commit_after>#!/usr/bin/perl
#
# mySociety/StringUtils.pm:
# Miscellaneous string handling utilities.
#
# Copyright (c) 2005 UK Citizens Online Democracy. All rights reserved.
# Email: <EMAIL>; WWW: http://www.mysociety.org/
#
# $Id: StringUtils.pm,v 1.1 2005-01-29 00:32:37 francis Exp $
#

package mySociety::StringUtils;

use strict;

BEGIN {
    use Exporter ();
    our @ISA = qw(Exporter);
    our @EXPORT_OK = qw(&trim &merge_spaces);
}
our @EXPORT_OK;

=head1 NAME

mySociety::StringUtils

=head1 DESCRIPTION

Various useful string related functions for applications.

=head1 FUNCTIONS

=over 4

=item random_bytes NUMBER

Return the given NUMBER of random bytes from /dev/random.

=item trim STRING

Remove whitespace from either end.

=cut
sub trim($) {
    ($_) = @_;
    s/^\s+//;     s/\s+$//;
    return $_;
}

=item merge_spaces STRING

Replace contiguous whitespace with a single space

=cut
sub merge_spaces($) {
    ($_) = @_;
    s/\s+/ /gs;
    return $_;
} 

=item break_into_lumps CONTENT

Converts HTML into space-trimmed, comment-removed lumps in an array.
Each lump was separated by a "paragraph" level tag.  Not just P, but also
e.g. TD or LI.

=cut
sub break_into_lumps($) {
    my ($content) = @_;

    $content =~ s/&amp;/&/g;  # make ampersands normal
    $content =~ s/<!--[^>]*-->//g;  # remove comments

    # Remove, without replacing with spaces, all the tags which are character
    # spanning.  We do this by removing all those except the most common
    # paragraph spanning tags.  This is because sometimes there are errors
    # where character spanning tags (e.g. a, strong, span...) are used in
    # the middle of a name.  e.g.
    #  D A Hal</a></span></span></strong><a href="../cllrhall_da.htm">l</a><br>
    $content =~ s/<\/?(?!p|br|tr|td|div|table|hr|li|ol|ul|h[1-9])\b[^>]+>//ig;

    # Flush all spacing (including NL CR) to just space
    $content = merge_spaces($content);

    # Break it up into parts between tags.  Note the only tags left are
    # those in the regular expression above at this point.
    my @lumps = split /<[^>]+>/, $content;
    @lumps = map { trim($_) } @lumps;
    return @lumps;
}

1;
{'fraction_non_alphanumeric': 0.13776408450704225, 'fraction_numerical': 0.011883802816901408, 'mean_word_length': 3.28060263653484, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4519642', 'n_tokens_mistral': 826, 'n_tokens_neox': 765, 'n_words': 254}