<commit_before># Normalizes expressions in a tagged source file.
# Arguments:
#    0: tag specification (regular expression).
#    1: expression tag.
#    2: column containing expressions to be normalized.
#    3: minimum score (0 or more) a proposed expression must have in order to be accepted
#        outright as an expression. Every proposed expression with a lower (or no) score is
#        to be replaced with the highest-scoring expression sharing its language variety and
#        degradation, if any such expression has a higher score than it.
#    4: minimum score a proposed expression that is not accepted outright as an expression,
#        or its replacement, must have in order to be accepted as an expression.
#    5: variety UID of expressions to be normalized.
#    6: tag of pre-normalized expression.
#    7: if proposed expressions not accepted as expressions and not having replacements accepted
#        as expressions are to be converted to definitions, definition tag, or blank if they
#        are to be converted to pre-normalized expressions.
#    8: regular expression matching the synonym delimiter if each proposed expression containing
#        such a delimiter is to be treated as a list of synonymous proposed expressions and
#        they are to be normalized if and only if all expressions in the list are
#        normalizable, or blank if not.

package PanLex::Serialize::normalize;

use warnings 'FATAL', 'all';
# Make every warning fatal.

use strict;
# Require strict checking of variable references, etc.

use utf8;
# Make Perl interpret the script as UTF-8 rather than bytes.

use PanLex;
use PanLex::Validation;

use Unicode::Normalize;
# Import the Unicode normalization module.

sub process {
    my ($in, $out, $tag, $extag, $excol, $minscore, $minscore_repl, $lv, $prenormtag, $dftag, $syndelim) = @_;

    foreach my $score ($minscore, $minscore_repl) {
        die "invalid minimum score" unless valid_int($score) && $score >= 0;
    }
    
    validate_col($excol);
    validate_uid($lv);
    
    my (%ex, %exok);

    my $lentag = length $extag;
    # Identify the length of the expression tag.

    my $done = 0;
    # Initialize the count of processed lines as 0.

    my @line = <$in>;
    # Identify a list of the lines of the input file.

    chomp @line;
    # Delete their trailing newlines.

    foreach my $line (@line) {
    # For each line:

        my @col = split /\t/, $line, -1;
        # Identify its columns.

        die "column $excol not present in line: $_" unless defined $col[$excol];

        if (length $col[$excol]) {
        # If the column containing proposed expressions is nonblank:

            my @seg = ($col[$excol] =~ /($tag.+?(?=$tag|$))/go);
            # Identify the tagged items, including tags, in it.

            foreach my $seg (@seg) {
            # For each of them:

                if (index($seg, $extag) == 0) {
                # If it is tagged as an expression:

                    foreach my $ex (PsList($seg, $lentag, $syndelim)) {
                    # For the expression, or for each expression if it is a pseudo-list:

                        $ex{$ex} = '';
                        # Add it to the table of proposed expression texts, if not
                        # already in it.
                    }
                }
            }
        }
    }

    my $result = panlex_query("/norm/$lv", { tt => [keys %ex] });
    die "could not retrieve normalization data from PanLex API: HTTP request failed" 
        unless $result;
    die "could not retrieve normalization data from PanLex API: $result->{error}" 
        unless $result->{status} eq 'OK';
    
    while (my ($tt,$norm) = each %{$result->{norm}}) {
        # For each proposed expression that has a score and whose score is sufficient for
        # outright acceptance as an expression:
        if ($norm->{score} >= $minscore) {
            $exok{$tt} = delete $ex{$tt};
        }
    }

    $result = panlex_query("/norm/$lv", { tt => [keys %ex], degrade => 1 });<commit_msg>disable caching for normalize queries<commit_after># Normalizes expressions in a tagged source file.
# Arguments:
#    0: tag specification (regular expression).
#    1: expression tag.
#    2: column containing expressions to be normalized.
#    3: minimum score (0 or more) a proposed expression must have in order to be accepted
#        outright as an expression. Every proposed expression with a lower (or no) score is
#        to be replaced with the highest-scoring expression sharing its language variety and
#        degradation, if any such expression has a higher score than it.
#    4: minimum score a proposed expression that is not accepted outright as an expression,
#        or its replacement, must have in order to be accepted as an expression.
#    5: variety UID of expressions to be normalized.
#    6: tag of pre-normalized expression.
#    7: if proposed expressions not accepted as expressions and not having replacements accepted
#        as expressions are to be converted to definitions, definition tag, or blank if they
#        are to be converted to pre-normalized expressions.
#    8: regular expression matching the synonym delimiter if each proposed expression containing
#        such a delimiter is to be treated as a list of synonymous proposed expressions and
#        they are to be normalized if and only if all expressions in the list are
#        normalizable, or blank if not.

package PanLex::Serialize::normalize;

use warnings 'FATAL', 'all';
# Make every warning fatal.

use strict;
# Require strict checking of variable references, etc.

use utf8;
# Make Perl interpret the script as UTF-8 rather than bytes.

use PanLex;
use PanLex::Validation;

use Unicode::Normalize;
# Import the Unicode normalization module.

sub process {
    my ($in, $out, $tag, $extag, $excol, $minscore, $minscore_repl, $lv, $prenormtag, $dftag, $syndelim) = @_;

    foreach my $score ($minscore, $minscore_repl) {
        die "invalid minimum score" unless valid_int($score) && $score >= 0;
    }
    
    validate_col($excol);
    validate_uid($lv);
    
    my (%ex, %exok);

    my $lentag = length $extag;
    # Identify the length of the expression tag.

    my $done = 0;
    # Initialize the count of processed lines as 0.

    my @line = <$in>;
    # Identify a list of the lines of the input file.

    chomp @line;
    # Delete their trailing newlines.

    foreach my $line (@line) {
    # For each line:

        my @col = split /\t/, $line, -1;
        # Identify its columns.

        die "column $excol not present in line: $_" unless defined $col[$excol];

        if (length $col[$excol]) {
        # If the column containing proposed expressions is nonblank:

            my @seg = ($col[$excol] =~ /($tag.+?(?=$tag|$))/go);
            # Identify the tagged items, including tags, in it.

            foreach my $seg (@seg) {
            # For each of them:

                if (index($seg, $extag) == 0) {
                # If it is tagged as an expression:

                    foreach my $ex (PsList($seg, $lentag, $syndelim)) {
                    # For the expression, or for each expression if it is a pseudo-list:

                        $ex{$ex} = '';
                        # Add it to the table of proposed expression texts, if not
                        # already in it.
                    }
                }
            }
        }
    }

    my $result = panlex_query("/norm/$lv", { tt => [keys %ex], cache => 0 });
    die "could not retrieve normalization data from PanLex API: HTTP request failed" 
        unless $result;
    die "could not retrieve normalization data from PanLex API: $result->{error}" 
        unless $result->{status} eq 'OK';
    
    while (my ($tt,$norm) = each %{$result->{norm}}) {
        # For each proposed expression that has a score and whose score is sufficient for
        # outright acceptance as an expression:
        if ($norm->{score} >= $minscore) {
            $exok{$tt} = delete $ex{$tt};
        }
    }

    $result = panlex_query("/norm/$lv", { tt => [keys %ex], degrade => 1, cache => 0 });
{'fraction_non_alphanumeric': 0.09735164296223639, 'fraction_numerical': 0.004659146640510054, 'mean_word_length': 2.4258714825703485, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3673667', 'n_tokens_mistral': 2298, 'n_tokens_neox': 2179, 'n_words': 978}