<commit_before># Tags metadata in a tab-delimited source file.
# Arguments:
#   col:   column containing metadata.
#   mdtag: metadatum tag. default '⫷md:gram⫸'.

package PanLex::Serialize::mdtag;
use strict;
use warnings 'FATAL', 'all';
use utf8;
use parent 'Exporter';
use PanLex::Validation;
use PanLex::Serialize::replace;
use PanLex::Serialize::dpptag;

our @EXPORT = qw/mdtag/;

my %VARMAP = (
);

sub mdtag {
    my $in = shift;
    my $out = shift;
    my $args = ref $_[0] ? $_[0] : \@_;

    my ($mdcol, $mdtag);
    
    if (ref $args eq 'HASH') {
        $mdcol    = $args->{col};
        $mdtag    = $args->{mdtag} // '⫷md:gram⫸';      
    } else {
        ($mdcol, $mdtag) = @$args;
    }
    validate_col($mdcol);

    my $var;

    if ($mdtag =~ /^⫷md:(.+)⫸$/) {
        die "don't know how to convert old md var: $1" unless exists $VARMAP{$1};
        $var = $VARMAP{$1};
    } else {
        die "don't know how to convert old mdtag: $mdtag";
    }

    my $temp;

    open my $fh, '>:encoding(utf8)', \$temp or die $!;
    replace($in, $fh, { cols => [$mdcol], from => '^', to => $var });
    close $fh;

    open $fh, '<:encoding(utf8)', \$temp or die $!;
    dpptag($fh, $out, { cols => [$mdcol] });
    close $fh;
}

1;<commit_msg>implement delim parameter in new mdtag<commit_after># Tags metadata in a tab-delimited source file.
# Arguments:
#   col:   column containing metadata.
#   mdtag: metadatum tag. default '⫷md:gram⫸'.
#   delim: metadatum delimiter, or '' if none. default ''.

package PanLex::Serialize::mdtag;
use strict;
use warnings 'FATAL', 'all';
use utf8;
use parent 'Exporter';
use PanLex::Validation;
use PanLex::Serialize::dpptag;

our @EXPORT = qw/mdtag/;

my %VARMAP = (
);

sub mdtag {
    my $in = shift;
    my $out = shift;
    my $args = ref $_[0] ? $_[0] : \@_;

    my ($mdcol, $mdtag, $delim);
    
    if (ref $args eq 'HASH') {
        $mdcol    = $args->{col};
        $mdtag    = $args->{mdtag} // '⫷md:gram⫸';
        $delim    = $args->{delim} // '';
    } else {
        ($mdcol, $mdtag) = @$args;
        $delim = '';
    }
    validate_col($mdcol);

    my $var;

    if ($mdtag =~ /^⫷md:(.+)⫸$/) {
        die "don't know how to convert old md var: $1" unless exists $VARMAP{$1};
        $var = $VARMAP{$1};
    } else {
        die "don't know how to convert old mdtag: $mdtag";
    }

    dpptag($fh, $out, { cols => [$mdcol], delim => $delim, prefix => $var });
}

1;
{'fraction_non_alphanumeric': 0.19342915811088296, 'fraction_numerical': 0.006570841889117043, 'mean_word_length': 2.566617862371889, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7032400', 'n_tokens_mistral': 994, 'n_tokens_neox': 951, 'n_words': 259}