<commit_before>package PEF::Log::Appender::File;
use base 'PEF::Log::Appender';
use IO::Handle;
use Carp;
use strict;
use warnings;

sub new {
	my ($class, %params) = @_;
	my $self = $class->SUPER::new(%params)->reload(\%params);
	truncate $self->{fh}, 0 if $params{init};
	$self;
}

sub reload {
	my ($self, $params) = @_;
	$self->_reload($params);
	my $out = $params->{out} or croak "no output file";
	return $self if exists $self->{fh} and $out eq $self->{out};
	open my $fh, ">>", $out or croak "can't open output file $out: $!";
	binmode $fh;
	$fh->autoflush;
	close $self->{fh} if $self->{fh};
	$self->{fh} = $fh;
	$self;
}

sub append {
	my ($self, $level, $stream, $msg) = @_;
	my $line = $self->SUPER::append($level, $stream, $msg);
	utf8::encode($line) if utf8::is_utf8($line);
	my $fh = $self->{fh};
	$fh->print($line);
	$fh->sync();
}

sub final {
	if ($_[0]->{fh}) {
		close $_[0]->{fh};
		undef $_[0]->{fh};
	}
}
<commit_msg>removed sync -- it really slows down the system<commit_after>package PEF::Log::Appender::File;
use base 'PEF::Log::Appender';
use IO::Handle;
use Carp;
use strict;
use warnings;

sub new {
	my ($class, %params) = @_;
	my $self = $class->SUPER::new(%params)->reload(\%params);
	truncate $self->{fh}, 0 if $params{init};
	$self;
}

sub reload {
	my ($self, $params) = @_;
	$self->_reload($params);
	my $out = $params->{out} or croak "no output file";
	return $self if exists $self->{fh} and $out eq $self->{out};
	open my $fh, ">>", $out or croak "can't open output file $out: $!";
	binmode $fh;
	$fh->autoflush;
	close $self->{fh} if $self->{fh};
	$self->{fh} = $fh;
	$self;
}

sub append {
	my ($self, $level, $stream, $msg) = @_;
	my $line = $self->SUPER::append($level, $stream, $msg);
	utf8::encode($line) if utf8::is_utf8($line);
	my $fh = $self->{fh};
	$fh->print($line);
}

sub final {
	if ($_[0]->{fh}) {
		close $_[0]->{fh};
		undef $_[0]->{fh};
	}
}

{'fraction_non_alphanumeric': 0.24552160168598525, 'fraction_numerical': 0.007376185458377239, 'mean_word_length': 4.425714285714286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7530006', 'n_tokens_mistral': 840, 'n_tokens_neox': 792, 'n_words': 204}