<commit_before>#!/usr/bin/perl

use strict;
use warnings;

use Data::Dumper;
use Dundi;

my $dundi = Dundi->new();

# should decode to
# VERSION=1
# EIDDIRECT=f23c91db245c
# CALLEDNUMBER=
# CALLEDCONTEXT=
# TTL=32
my $stream = '0a0200011704726172611604726172610406f23c91db245c0303323030020470726976060200200e2403556e656e6372797074656420726573706f6e736573206e6f74207065726d6974746564';
$stream = pack('H*', $stream);

my $ie = $dundi->parse_ie('', $stream);
print Dumper($ie);<commit_msg>add a new option to decode encdata<commit_after>#!/usr/bin/perl

use strict;
use warnings;

use Data::Dumper;
use Dundi;

my $dundi = Dundi->new();

# should decode to
# VERSION=1
# EIDDIRECT=f23c91db245c
# CALLEDNUMBER=
# CALLEDCONTEXT=
# TTL=32
#my $stream = '0a0200011704726172611604726172610406f23c91db245c0303323030020470726976060200200e2403556e656e6372797074656420726573706f6e736573206e6f74207065726d6974746564';

# should decode to
# ...
my $stream = '0106f23c91db245c13043053e37e1040e6407a2374d2b64ae2bc2944a15a0a548208bdf5b2f902b54853940f4c11cdc9054fea8395dd793597d15e90f1d5821367f2a95568c142b4b5dded761d9fb9ad';

$stream = pack('H*', $stream);

my $ie = $dundi->parse_ie('', $stream);
print Dumper($ie);
{'fraction_non_alphanumeric': 0.11416666666666667, 'fraction_numerical': 0.34, 'mean_word_length': 8.685483870967742, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5127917', 'n_tokens_mistral': 779, 'n_tokens_neox': 553, 'n_words': 75}