<commit_before># boxes-lib.pl
# Functions to parsing user mail files

use POSIX;
if ($userconfig{'date_tz'} || $config{'date_tz'}) {
        # Set the timezone for all date calculations, and force a conversion
        # now as in some cases the first one fails!
        $ENV{'TZ'} = $userconfig{'date_tz'} ||
                     $config{'date_tz'};
        strftime('%H:%M', localtime(time()));
        }
use Time::Local;

$dbm_index_min = 1000000;
$dbm_index_version = 3;

# list_mails(user|file, [start], [end])
# Returns a subset of mail from a mbox format file
sub list_mails
{
local (@rv, $h, $done);
my %index;
my $umf = &user_mail_file($_[0]);
&open_as_mail_user(MAIL, $umf) || &error("Failed to open $umf : $!");
&build_dbm_index($_[0], \%index);
local ($start, $end);
local $isize = $index{'mailcount'};
if (@_ == 1 || !defined($_[1]) && !defined($_[2])) {
	$start = 0; $end = $isize-1;
	}
elsif ($_[2] < 0) {
	$start = $isize+$_[2]-1; $end = $isize+$_[1]-1;
	$start = $start<0 ? 0 : $start;
	}
else {
	$start = $_[1]; $end = $_[2];
	$end = $isize-1 if ($end >= $isize);
	}
$rv[$isize-1] = undef if ($isize);	# force array to right size
local $dash = &dash_mode($_[0]);
$start = 0 if ($start < 0);
for($i=$start; $i<=$end; $i++) {
	# Seek to mail position
	local @idx = split(/\0/, $index{$i});
	local $pos = $idx[0];
	local $startline = $idx[1];
	seek(MAIL, $pos, 0);
	print STDERR "seek to $pos err $!\n";

	# Read the mail
	local $mail = &read_mail_fh(MAIL, $dash ? 2 : 1, 0);
	$mail->{'line'} = $startline;
	$mail->{'eline'} = $startline + $mail->{'lines'} - 1;
	$mail->{'idx'} = $i;
	# ID is position in file and message ID
	$mail->{'id'} = $pos." ".$i." ".$startline." ".
		substr($mail->{'header'}->{'message-id'}, 0, 255);<commit_msg>Remove debug message<commit_after># boxes-lib.pl
# Functions to parsing user mail files

use POSIX;
if ($userconfig{'date_tz'} || $config{'date_tz'}) {
        # Set the timezone for all date calculations, and force a conversion
        # now as in some cases the first one fails!
        $ENV{'TZ'} = $userconfig{'date_tz'} ||
                     $config{'date_tz'};
        strftime('%H:%M', localtime(time()));
        }
use Time::Local;

$dbm_index_min = 1000000;
$dbm_index_version = 3;

# list_mails(user|file, [start], [end])
# Returns a subset of mail from a mbox format file
sub list_mails
{
local (@rv, $h, $done);
my %index;
my $umf = &user_mail_file($_[0]);
&open_as_mail_user(MAIL, $umf) || &error("Failed to open $umf : $!");
&build_dbm_index($_[0], \%index);
local ($start, $end);
local $isize = $index{'mailcount'};
if (@_ == 1 || !defined($_[1]) && !defined($_[2])) {
	$start = 0; $end = $isize-1;
	}
elsif ($_[2] < 0) {
	$start = $isize+$_[2]-1; $end = $isize+$_[1]-1;
	$start = $start<0 ? 0 : $start;
	}
else {
	$start = $_[1]; $end = $_[2];
	$end = $isize-1 if ($end >= $isize);
	}
$rv[$isize-1] = undef if ($isize);	# force array to right size
local $dash = &dash_mode($_[0]);
$start = 0 if ($start < 0);
for($i=$start; $i<=$end; $i++) {
	# Seek to mail position
	local @idx = split(/\0/, $index{$i});
	local $pos = $idx[0];
	local $startline = $idx[1];
	seek(MAIL, $pos, 0);

	# Read the mail
	local $mail = &read_mail_fh(MAIL, $dash ? 2 : 1, 0);
	$mail->{'line'} = $startline;
	$mail->{'eline'} = $startline + $mail->{'lines'} - 1;
	$mail->{'idx'} = $i;
	# ID is position in file and message ID
	$mail->{'id'} = $pos." ".$i." ".$startline." ".
		substr($mail->{'header'}->{'message-id'}, 0, 255);
{'fraction_non_alphanumeric': 0.2318548387096774, 'fraction_numerical': 0.024193548387096774, 'mean_word_length': 3.8102493074792245, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5965950', 'n_tokens_mistral': 1441, 'n_tokens_neox': 1370, 'n_words': 370}