<commit_before>package Zeta::Pack::8583;

use strict;
use warnings;

use IO::File;
use Data::Dump;
use Zeta::Log;
use integer;
use Encode;

use constant  ID_IDX    => 0;
use constant  TYPE_IDX  => 1;
use constant  LEN_IDX   => 2;
use constant  CLASS_IDX => 3;
use constant  LENC_IDX  => 4;
use constant  DENC_IDX  => 5;

#
#  conf  =>  $xxx/conf/tlpos.conf
#
sub new {
    my $self = bless {}, shift;
    $self->_init(@_);
    return $self;
}

#
#  conf  => /path/of/config/file
#  conf  => \*DATA
#
sub _init {

    my $self = shift;
    my $args = { @_ };  
    
    # load the config file
    my $conf_data = [];
    my $fh;
    if ('GLOB' eq ref $args->{conf}) {
        $fh = $args->{conf};
    }
    else {
        $fh = IO::File->new("< $args->{conf}");
    }
    unless($fh) {
        die "config file : $args->{conf} not exist";
        exit;
    }
    while (<$fh>) {
        unless (/^#/) {
            # remove the first empty element
            s/^\s+//;
            my $data = [split /\s+/, $_];
            # store the first 5 elements
            $conf_data->[@{$data}[0]] = [@{$data}[0..5]];
        }
    }
    for (my $i = 0; $i < @$conf_data; ++$i) {
      $_=$conf_data->[$i];
    }
    $fh->close;
    $self->{config} = $conf_data;
    $self->{hdr}    = $args->{hdr};
}

sub pack {
    
    my $self   = shift;
    my $fld    = shift;<commit_msg>Zeta::Pack::8583 add FH support<commit_after>package Zeta::Pack::8583;

use strict;
use warnings;

use IO::File;
use Data::Dump;
use Zeta::Log;
use integer;
use Encode;

use constant  ID_IDX    => 0;
use constant  TYPE_IDX  => 1;
use constant  LEN_IDX   => 2;
use constant  CLASS_IDX => 3;
use constant  LENC_IDX  => 4;
use constant  DENC_IDX  => 5;

#
#  conf  =>  $xxx/conf/tlpos.conf
#
sub new {
    my $self = bless {}, shift;
    $self->_init(@_);
    return $self;
}

#
#  conf  => /path/of/config/file
#  conf  => \*DATA
#
sub _init {

    my $self = shift;
    my $args = { @_ };  
    
    # load the config file
    my $conf_data = [];
    my $fh;
    if ('GLOB' eq ref $args->{conf}) {
        $fh = $args->{conf};
    }
    else {
        $fh = IO::File->new("< $args->{conf}");
        unless($fh) {
            die "config file : $args->{conf} not exist";
            exit;
        }
    }
    while (<$fh>) {
        unless (/^#/) {
            # remove the first empty element
            s/^\s+//;
            my $data = [split /\s+/, $_];
            # store the first 5 elements
            $conf_data->[@{$data}[0]] = [@{$data}[0..5]];
        }
    }
    for (my $i = 0; $i < @$conf_data; ++$i) {
      $_=$conf_data->[$i];
    }
    $fh->close;
    $self->{config} = $conf_data;
    $self->{hdr}    = $args->{hdr};
}

sub pack {
    
    my $self   = shift;
    my $fld    = shift;
{'fraction_non_alphanumeric': 0.1919191919191919, 'fraction_numerical': 0.012265512265512266, 'mean_word_length': 1.8765560165975104, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700180', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1060, 'n_words': 262}