<commit_before>package PEF::Front::Request;
use strict;
use warnings;
use JSON;
use Carp ();
use Encode;
use PEF::Front::Headers;
use PEF::Front::File;

sub new {
	my ($class, $env) = @_;
	Carp::croak(q{$env is required})
	  unless defined $env && ref ($env) eq 'HASH';
	my $self = bless {env => $env}, $class;
	$self->parse;
	$self;
}
sub env              { $_[0]->{env} }
sub remote_ip        { $_[0]->{env}{REMOTE_ADDR} }
sub protocol         { $_[0]->{env}{SERVER_PROTOCOL} }
sub method           { $_[0]->{env}{REQUEST_METHOD} }
sub port             { $_[0]->{env}{SERVER_PORT} }
sub user             { $_[0]->{env}{REMOTE_USER} }
sub request_uri      { $_[0]->{env}{REQUEST_URI} }
sub path_info        { $_[0]->{env}{PATH_INFO} }
sub path             { $_[0]->{env}{PATH_INFO} || '/' }
sub query_string     { $_[0]->{env}{QUERY_STRING} }
sub script_name      { $_[0]->{env}{SCRIPT_NAME} }
sub scheme           { $_[0]->{env}{'psgi.url_scheme'} }
sub secure           { $_[0]->scheme eq 'https' }
sub input            { $_[0]->{env}{'psgi.input'} }
sub content_length   { $_[0]->{env}{CONTENT_LENGTH} }
sub content_type     { $_[0]->{env}{CONTENT_TYPE} }
sub raw_body         { $_[0]->{raw_body} }
sub content_encoding { $_[0]->headers->get_header("content_encoding") }
sub header           { $_[0]->headers->get_header($_[1]) }
sub referer          { $_[0]->headers->get_header("referer") }
sub user_agent       { $_[0]->headers->get_header("user_agent") }

sub logger {
	$_[0]->{env}{'psgix.logger'} || sub { }
}

sub parse {
	my $self = $_[0];
	$self->_parse_query_params;
	$self->_parse_request_body if $self->method eq 'POST';
}

sub params {
	my $self = $_[0];
	return $self->{params} if exists $self->{params};
	my $q = $self->{query_params} || {};
	my $p = $self->{body_params}  || {};
	$self->{params} = {%$q, %$p};
}

sub param {
	my ($self, $param) = @_;
	return $self->params->{$param};
}

sub hostname {
	my $self = $_[0];
	return $self->{hostname} if exists $self->{hostname};
	$self->{hostname} = $self->{env}{HTTP_HOST} || $self->{env}{SERVER_NAME};
	$self->{hostname} =~ s/:\d+//;
	$self->{hostname};
}

sub base {
	my $self = $_[0];
	return $self->{base} if exists $self->{base};
	$self->{base} =
	  $self->scheme . "://" . ($self->{env}{HTTP_HOST} || $self->{env}{SERVER_NAME}) . $self->request_uri;
	$self->{base};
}

sub cookies {
	my $self = $_[0];
	return {} unless $self->{env}{HTTP_COOKIE};
	return $self->{cookies} if exists $self->{cookies};
	my %results;
	my @pairs = grep m/=/, split "[;,] ?", $self->{env}{HTTP_COOKIE};
	for my $pair (@pairs) {
		$pair =~ s/^\s+//;
		$pair =~ s/\s+$//;
		my ($key, $value) =
		  map { tr/+/ /; s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; decode_utf8 $_ } split ("=", $pair, 2);
		$results{$key} = $value;
	}
	$self->{cookies} = \%results;
}

sub _parse_urlencoded {
	my $query = $_[0];
	my $form  = {};
	my @pairs = split (/[&;]/, $query);
	foreach my $pair (@pairs) {
		my ($name, $value) =
		  map { tr/+/ /; s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; $_ }
		  split (/=/, $pair, 2);
		eval { $form->{decode_utf8($name)} = decode_utf8 $value if $name };
	}
	return $form;
}

sub headers {
	my $self = shift;
	if (!defined $self->{headers}) {
		my $env = $self->{env};
		$self->{headers} = PEF::Front::HTTPHeaders->new(
			map {
				(my $field = decode_utf8 $_) =~ s/^https?_//;
				($field => decode_utf8 $env->{$_});
			  }
			  grep {
				/^HTTP/ || /^CONTENT/
			  } keys %$env
		);
	}
	$self->{headers};
}

sub _parse_query_params {
	my $self = $_[0];
	return {} if !$self->{env}{QUERY_STRING};
	return $self->{query_params} if exists $self->{query_params};
	$self->{query_params} = _parse_urlencoded($self->{env}{QUERY_STRING});
}

sub _parse_request_body {
	my $self = $_[0];
	my $ct   = $self->{env}{CONTENT_TYPE};
	my $cl   = $self->{env}{CONTENT_LENGTH};
	if (!$ct && !$cl) {
		return;
	}
	return $self->{body_params} if exists $self->{body_params};
	if (index ($ct, 'application/x-www-form-urlencoded') == 0) {
		$self->{raw_body} = '';
		my $buffer;
		while ($cl && $self->input->read($buffer, $cl)) {
			$self->{raw_body} .= $buffer;
			$cl -= length $buffer;
		}
		$self->{body_params} = _parse_urlencoded($self->{raw_body});
	} elsif (index ($ct, 'application/json') == 0) {
		$self->{raw_body} = '';
		my $buffer;
		while ($cl && $self->input->read($buffer, $cl)) {
			$self->{raw_body} .= $buffer;
			$cl -= length $buffer;
		}
		my $from_json = $self->{raw_body};
		if (substr ($self->{raw_body}, 0, 2) eq '%7') {
			$from_json =~ tr/+/ /;
			$from_json =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg;
		}
		$self->{body_params} = eval { decode_json $from_json} || {};
		return $self->{body_params};
	} elsif (index ($ct, 'multipart/form-data') == 0) {
		$self->_parse_multipart_form;
	}
	if (exists $self->{body_params}{json}) {
		my $form = eval { decode_json $self->{body_params}{json} } || {};
		$self->{body_params} = {%{$self->{body_params}}, %$form};
	}
	return $self->{body_params};
}

sub _parse_multipart_form {
	my $self   = $_[0];
	my $buffer = '';
	my $input  = $self->input;
	my $cl     = $self->{env}{CONTENT_LENGTH};
	my $ct     = $self->{env}{CONTENT_TYPE};
	$self->{body_params} = {};
	my $form = $self->{body_params};
	my ($boundary) = $ct =~ /boundary=\"?([^\";,]+)\"?/;
	return if not defined $boundary;
	my $start_boundary = "--" . $boundary . "\x0d\x0a";
	my $end_boundary   = "--" . $boundary . "--\x0d\x0a";
	my $chl            = 65536;
	my $lchnk          = 0;
	my $current_field;

	while (1) {
		my $chunk;
		my $start = index ($buffer, $start_boundary);
		$start = index ($buffer, $end_boundary) if $start == -1;
		$buffer .= $chunk if $start == -1 && $input->read($chunk, $chl);
		last if $buffer eq '';<commit_msg>fixed double utf8 decoding for "json" form field<commit_after>package PEF::Front::Request;
use strict;
use warnings;
use JSON;
use Carp ();
use Encode;
use PEF::Front::Headers;
use PEF::Front::File;

sub new {
	my ($class, $env) = @_;
	Carp::croak(q{$env is required})
	  unless defined $env && ref ($env) eq 'HASH';
	my $self = bless {env => $env}, $class;
	$self->parse;
	$self;
}
sub env              { $_[0]->{env} }
sub remote_ip        { $_[0]->{env}{REMOTE_ADDR} }
sub protocol         { $_[0]->{env}{SERVER_PROTOCOL} }
sub method           { $_[0]->{env}{REQUEST_METHOD} }
sub port             { $_[0]->{env}{SERVER_PORT} }
sub user             { $_[0]->{env}{REMOTE_USER} }
sub request_uri      { $_[0]->{env}{REQUEST_URI} }
sub path_info        { $_[0]->{env}{PATH_INFO} }
sub path             { $_[0]->{env}{PATH_INFO} || '/' }
sub query_string     { $_[0]->{env}{QUERY_STRING} }
sub script_name      { $_[0]->{env}{SCRIPT_NAME} }
sub scheme           { $_[0]->{env}{'psgi.url_scheme'} }
sub secure           { $_[0]->scheme eq 'https' }
sub input            { $_[0]->{env}{'psgi.input'} }
sub content_length   { $_[0]->{env}{CONTENT_LENGTH} }
sub content_type     { $_[0]->{env}{CONTENT_TYPE} }
sub raw_body         { $_[0]->{raw_body} }
sub content_encoding { $_[0]->headers->get_header("content_encoding") }
sub header           { $_[0]->headers->get_header($_[1]) }
sub referer          { $_[0]->headers->get_header("referer") }
sub user_agent       { $_[0]->headers->get_header("user_agent") }

sub logger {
	$_[0]->{env}{'psgix.logger'} || sub { }
}

sub parse {
	my $self = $_[0];
	$self->_parse_query_params;
	$self->_parse_request_body if $self->method eq 'POST';
}

sub params {
	my $self = $_[0];
	return $self->{params} if exists $self->{params};
	my $q = $self->{query_params} || {};
	my $p = $self->{body_params}  || {};
	$self->{params} = {%$q, %$p};
}

sub param {
	my ($self, $param) = @_;
	return $self->params->{$param};
}

sub hostname {
	my $self = $_[0];
	return $self->{hostname} if exists $self->{hostname};
	$self->{hostname} = $self->{env}{HTTP_HOST} || $self->{env}{SERVER_NAME};
	$self->{hostname} =~ s/:\d+//;
	$self->{hostname};
}

sub base {
	my $self = $_[0];
	return $self->{base} if exists $self->{base};
	$self->{base} =
	  $self->scheme . "://" . ($self->{env}{HTTP_HOST} || $self->{env}{SERVER_NAME}) . $self->request_uri;
	$self->{base};
}

sub cookies {
	my $self = $_[0];
	return {} unless $self->{env}{HTTP_COOKIE};
	return $self->{cookies} if exists $self->{cookies};
	my %results;
	my @pairs = grep m/=/, split "[;,] ?", $self->{env}{HTTP_COOKIE};
	for my $pair (@pairs) {
		$pair =~ s/^\s+//;
		$pair =~ s/\s+$//;
		my ($key, $value) =
		  map { tr/+/ /; s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; decode_utf8 $_ } split ("=", $pair, 2);
		$results{$key} = $value;
	}
	$self->{cookies} = \%results;
}

sub _parse_urlencoded {
	my $query = $_[0];
	my $form  = {};
	my @pairs = split (/[&;]/, $query);
	foreach my $pair (@pairs) {
		my ($name, $value) =
		  map { tr/+/ /; s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; $_ }
		  split (/=/, $pair, 2);
		eval { $form->{decode_utf8($name)} = decode_utf8 $value if $name };
	}
	return $form;
}

sub headers {
	my $self = shift;
	if (!defined $self->{headers}) {
		my $env = $self->{env};
		$self->{headers} = PEF::Front::HTTPHeaders->new(
			map {
				(my $field = decode_utf8 $_) =~ s/^https?_//;
				($field => decode_utf8 $env->{$_});
			  }
			  grep {
				/^HTTP/ || /^CONTENT/
			  } keys %$env
		);
	}
	$self->{headers};
}

sub _parse_query_params {
	my $self = $_[0];
	return {} if !$self->{env}{QUERY_STRING};
	return $self->{query_params} if exists $self->{query_params};
	$self->{query_params} = _parse_urlencoded($self->{env}{QUERY_STRING});
}

sub _parse_request_body {
	my $self = $_[0];
	my $ct   = $self->{env}{CONTENT_TYPE};
	my $cl   = $self->{env}{CONTENT_LENGTH};
	if (!$ct && !$cl) {
		return;
	}
	return $self->{body_params} if exists $self->{body_params};
	if (index ($ct, 'application/x-www-form-urlencoded') == 0) {
		$self->{raw_body} = '';
		my $buffer;
		while ($cl && $self->input->read($buffer, $cl)) {
			$self->{raw_body} .= $buffer;
			$cl -= length $buffer;
		}
		$self->{body_params} = _parse_urlencoded($self->{raw_body});
	} elsif (index ($ct, 'application/json') == 0) {
		$self->{raw_body} = '';
		my $buffer;
		while ($cl && $self->input->read($buffer, $cl)) {
			$self->{raw_body} .= $buffer;
			$cl -= length $buffer;
		}
		my $from_json = $self->{raw_body};
		if (substr ($self->{raw_body}, 0, 2) eq '%7') {
			$from_json =~ tr/+/ /;
			$from_json =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg;
		}
		$self->{body_params} = eval { decode_json $from_json} || {};
		return $self->{body_params};
	} elsif (index ($ct, 'multipart/form-data') == 0) {
		$self->_parse_multipart_form;
	}
	if (exists $self->{body_params}{json}) {
		my $form = eval { from_json $self->{body_params}{json} } || {};
		$self->{body_params} = {%{$self->{body_params}}, %$form};
	}
	return $self->{body_params};
}

sub _parse_multipart_form {
	my $self   = $_[0];
	my $buffer = '';
	my $input  = $self->input;
	my $cl     = $self->{env}{CONTENT_LENGTH};
	my $ct     = $self->{env}{CONTENT_TYPE};
	$self->{body_params} = {};
	my $form = $self->{body_params};
	my ($boundary) = $ct =~ /boundary=\"?([^\";,]+)\"?/;
	return if not defined $boundary;
	my $start_boundary = "--" . $boundary . "\x0d\x0a";
	my $end_boundary   = "--" . $boundary . "--\x0d\x0a";
	my $chl            = 65536;
	my $lchnk          = 0;
	my $current_field;

	while (1) {
		my $chunk;
		my $start = index ($buffer, $start_boundary);
		$start = index ($buffer, $end_boundary) if $start == -1;
		$buffer .= $chunk if $start == -1 && $input->read($chunk, $chl);
		last if $buffer eq '';
{'fraction_non_alphanumeric': 0.24612184764710981, 'fraction_numerical': 0.012739405494410262, 'mean_word_length': 3.367903103709311, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 205, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3794263', 'n_tokens_mistral': 4904, 'n_tokens_neox': 4694, 'n_words': 935}