<commit_before># Normalizes expressions in a tagged source file.
# Arguments:
#   col:      column containing expressions to be normalized.
#   uid:      variety UID of expressions to be normalized.
#   min:      minimum score (0 or more) a proposed expression must have in order 
#               to be accepted outright as an expression. Every proposed 
#               expression with a lower (or no) score is to be replaced with the 
#               highest-scoring expression sharing its language variety and 
#               degradation, if any such expression has a higher score than it.
#   mindeg:   minimum score a proposed expression that is not accepted outright 
#               as an expression, or its replacement, must have in order to be
#               accepted as an expression.
#   failtag:  tag with which to retag proposed expressions not accepted as 
#               expressions and not having replacements accepted as expressions; 
#               '' (blank) if they are to be converted to pre-normalized 
#               expressions. default '⫷df⫸'.
#   ignore:   regex matching expressions to be ignored in normalization; or ''
#               (blank) if none. default ''.
#   delim:    regex matching the synonym delimiter, if each proposed expression
#               containing such a delimiter is to be treated as a list of
#               synonymous proposed expressions and they are to be normalized if
#               and only if all expressions in the list are normalizable; or ''
#               (blank) if not. default ''. example: ', '.
#   extag:    expression tag. default '⫷ex⫸'.
#   exptag:   pre-normalized expression tag. default '⫷exp⫸'.
#   tagre:    regex identifying any tag. default '⫷[a-z:]+⫸'.

package PanLex::Serialize::normalize;

use warnings 'FATAL', 'all';
# Make every warning fatal.

use strict;
# Require strict checking of variable references, etc.

use utf8;
# Make Perl interpret the script as UTF-8 rather than bytes.

use PanLex;
use PanLex::Validation;

use Unicode::Normalize;
# Import the Unicode normalization module.

sub process {
    my $in = shift;
    my $out = shift;
    my $args = ref $_[0] ? $_[0] : \@_;
    
    my ($excol, $uid, $min, $mindeg, $failtag, $ignore, $delim, $extag, $exptag, $tagre);
    
    if (ref $args eq 'HASH') {
        $args->{failtag} = $args->{dftag} unless defined $args->{failtag};

        $excol    = $args->{col};
        $uid      = $args->{uid};
        $min      = $args->{min};
        $mindeg   = $args->{mindeg};
        $failtag  = defined $args->{failtag} ? $args->{failtag} : '⫷df⫸';
        $ignore   = defined $args->{ignore} ? $args->{ignore} : '';
        $delim    = defined $args->{delim} ? $args->{delim} : '';
        $extag    = defined $args->{extag} ? $args->{extag} : '⫷ex⫸';
        $exptag   = defined $args->{exptag} ? $args->{exptag} : '⫷exp⫸';
        $tagre    = defined $args->{tagre} ? $args->{tagre} : '⫷[a-z:]+⫸';      
    } else {
        ($tagre, $extag, $excol, $min, $mindeg, $uid, $exptag, $failtag, $delim) = @$args;
        $ignore = '';
    }

    validate_col($excol);
    validate_uid($uid);

    foreach my $score ($min, $mindeg) {<commit_msg>make normalize not alter $args<commit_after># Normalizes expressions in a tagged source file.
# Arguments:
#   col:      column containing expressions to be normalized.
#   uid:      variety UID of expressions to be normalized.
#   min:      minimum score (0 or more) a proposed expression must have in order 
#               to be accepted outright as an expression. Every proposed 
#               expression with a lower (or no) score is to be replaced with the 
#               highest-scoring expression sharing its language variety and 
#               degradation, if any such expression has a higher score than it.
#   mindeg:   minimum score a proposed expression that is not accepted outright 
#               as an expression, or its replacement, must have in order to be
#               accepted as an expression.
#   failtag:  tag with which to retag proposed expressions not accepted as 
#               expressions and not having replacements accepted as expressions; 
#               '' (blank) if they are to be converted to pre-normalized 
#               expressions. default '⫷df⫸'.
#   ignore:   regex matching expressions to be ignored in normalization; or ''
#               (blank) if none. default ''.
#   delim:    regex matching the synonym delimiter, if each proposed expression
#               containing such a delimiter is to be treated as a list of
#               synonymous proposed expressions and they are to be normalized if
#               and only if all expressions in the list are normalizable; or ''
#               (blank) if not. default ''. example: ', '.
#   extag:    expression tag. default '⫷ex⫸'.
#   exptag:   pre-normalized expression tag. default '⫷exp⫸'.
#   tagre:    regex identifying any tag. default '⫷[a-z:]+⫸'.

package PanLex::Serialize::normalize;

use warnings 'FATAL', 'all';
# Make every warning fatal.

use strict;
# Require strict checking of variable references, etc.

use utf8;
# Make Perl interpret the script as UTF-8 rather than bytes.

use PanLex;
use PanLex::Validation;

use Unicode::Normalize;
# Import the Unicode normalization module.

sub process {
    my $in = shift;
    my $out = shift;
    my $args = ref $_[0] ? $_[0] : \@_;
    
    my ($excol, $uid, $min, $mindeg, $failtag, $ignore, $delim, $extag, $exptag, $tagre);
    
    if (ref $args eq 'HASH') {
        $excol    = $args->{col};
        $uid      = $args->{uid};
        $min      = $args->{min};
        $mindeg   = $args->{mindeg};
        $failtag  = defined $args->{failtag} ? $args->{failtag} : defined $args->{dftag} ? $args->{dftag} : '⫷df⫸';
        $ignore   = defined $args->{ignore} ? $args->{ignore} : '';
        $delim    = defined $args->{delim} ? $args->{delim} : '';
        $extag    = defined $args->{extag} ? $args->{extag} : '⫷ex⫸';
        $exptag   = defined $args->{exptag} ? $args->{exptag} : '⫷exp⫸';
        $tagre    = defined $args->{tagre} ? $args->{tagre} : '⫷[a-z:]+⫸';      
    } else {
        ($tagre, $extag, $excol, $min, $mindeg, $uid, $exptag, $failtag, $delim) = @$args;
        $ignore = '';
    }

    validate_col($excol);
    validate_uid($uid);

    foreach my $score ($min, $mindeg) {
{'fraction_non_alphanumeric': 0.12560841576385617, 'fraction_numerical': 0.0015701051970482022, 'mean_word_length': 2.458197611292074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1431950', 'n_tokens_mistral': 1988, 'n_tokens_neox': 1949, 'n_words': 694}