<commit_before>package Combust::Control;
use strict;
use Exception::Class ('ControllerException');
use Combust::Constant qw(OK SERVER_ERROR MOVED DONE DECLINED REDIRECT);
use Carp qw(confess cluck carp);
use Digest::SHA1 qw(sha1_hex);
use URI::Escape qw(uri_escape);
use Encode qw(encode_utf8);
use base qw(Combust::Redirect);

# TODO: figure out why we use this; remove it if possible
require bytes;

use Exception::Class ('Ex_ServerError');

use Combust::Cache;
use Combust::Template;
use Combust::Cookies;
use Combust::Secret qw(get_secret);

use Combust::Config;

my $config = Combust::Config->new();

sub config { $config }

my $root = $config->root;

sub r {
  my $self = shift;
  # some day we'll deprecate this - it only works for Apache13 and Apache2
  return $self->request->_r;
}

sub req_param {
  my $self = shift;
  $self->request->req_param(@_);
}

sub param  { cluck "param() deprecated; use tpl_param()"; tpl_param(@_) }
sub params { cluck "params() deprecated; use tpl_params()"; tpl_params(@_) }

sub tpl_param {
  my ($self, $key) = (shift, shift);
  return unless $key;
  #Carp::cluck "param('$key' ...) called" if $key eq "user";
  $self->{params}->{$key} = shift if @_;
  return $self->{params}->{$key};
}

sub tpl_params {
  my $self = shift;
  cluck("tpl_params called with [$self] as self.  Did you configure the handler to call ->handler instead of ->super?")
    unless ref $self;
  cluck('Combust::Control->tpl_params called with parameters, did you mean to call "param"?') if @_;
  $self->{params} || {};
}

sub new {
  my ($class, $r) = @_;

  # return if we are already blessed
  return $class if ref $class;

  my $self = bless( { } , $class);
  
  $self;
}

sub super ($$) {
  my $class   = shift;
  my $r = shift;

  confess(__PACKAGE__ . '->super got called without $r') unless $r;
  return unless $r;

  my $self = $class->new($r);

  Combust::Notes::handler($r);

  $self->tt->set_include_path($self->get_include_path);

  my $status;

  eval {
    $status = OK;
    $status = $self->init if $self->can('init');
  };
  if ($@) {
    cluck "$self->init died: $@";
    return SERVER_ERROR;
  }
  return $status unless $status == OK;

  eval {
      ($status) = $self->handler($self->r);
  };
  cluck "Combust::Control: oops, class handler died with: $@" if $@;
  return SERVER_ERROR if $@;

  # should we do this to make it harder for people to shoot themselves in the foot?
  # $self->_cleanup_params;

  return $status;
}

sub handler {
  my $self = shift;
  unless ($self->can('render')) {
    my $msg = "$self doesn't have a render method; you probably got the inheritance order messed up somewhere.";
    warn $msg;
    die $msg;
  }

  my $redir_status = $self->redirect_check;
  return $redir_status unless $redir_status == DECLINED;

  my ($status, $output, $content_type) = $self->do_request();
  # have to return 'OK' and fake it with r->status or some such to make a custom 404 easily
  return $status unless $status == OK;
  my @r = $self->send_output($output, $content_type);
  if ($self->can('cleanup')) {
      eval { $self->cleanup };
      warn "CLEANUP method failed: $@" if $@; 
  }
  return @r;
}

sub do_request {
  my $self = shift;

  my $cache_info = $self->cache_info || {};

  my ($status, $output, $cache);

  if ($cache_info->{id} 
      && ($cache = Combust::Cache->new( type => ($cache_info->{type} || '') ))
     ) {
    my $cache_data;
    $cache_data = $cache->fetch(id => $cache_info->{id})
      unless $self->req_param('cache_bypass');

    if ($cache_data and $cache_data->{data}) {
      $self->post_process($cache_data->{data});
      $self->r->update_mtime($cache_data->{created_timestamp});
      my ($content_type);
      $content_type = $cache->{meta_data}->{content_type}
	if $cache->{meta_data}->{content_type};
      $status = $cache->{meta_data}->{status}
	if $cache->{meta_data}->{status};

      $status ||= OK;

      return ($status, $cache_data->{data}, $content_type);
    }
  }

  ($status, $output, my $content_type) = eval { $self->render };
  if ($@) {
      warn "render failed: $@";
      $status = SERVER_ERROR;
  }
  return $status unless $status == OK;

  # sometimes we end up here with "OK" but with no content ... gah.
  if ($cache and $output and $status != SERVER_ERROR and !$self->no_cache) {
    $cache_info->{meta_data}->{content_type} = $content_type if $content_type;
    $cache_info->{meta_data}->{status}       = $status || $self->r->status;
    $cache->store( %$cache_info, data => $output );
  }
  
  $status = $self->post_process($output);

  return ($status, $output, $content_type);
}

sub no_cache {
    my $self   = shift;
    my $status = shift;
    $self->{no_cache} = $status if defined $status;
    return $self->{no_cache};
}

sub cache_info {}
sub post_process { OK }

sub _cleanup_params {
  my $self = shift;
  for my $param (keys %{$self->{params}}) {
    delete $self->{params}->{$param};
  }
}

sub get_include_path {
  my $self = shift;

  my $r = $self->r;

  my $site = $self->site;
  unless ($site) {
    my @path = ("$root/apache/root_templates/");
    unshift @path, $r->document_root if $r->dir_config('UseDocumentRoot');
    return \@path;
  }

  my @site_dirs = split /:/, ($config->site->{$site}->{docs_site} || $site);

  #warn Data::Dumper->Dump([\$r], [qw(r)]);

  my $cookies = $self->cookies;

  #warn "param:root: ", $r->param('root');
  #warn "root coookie : ", $cookies->cookie('root');

  my ($user);
  my $root_param = $r->param('root') || '';
  if (($user) = ($root_param =~ m!^/?([a-zA-Z]+)$!)) {
    $cookies->cookie('root', "$user");
  } 
  elsif ($root_param eq "/") {
    # don't set user, reset the cookie
    $cookies->cookie('root', "/");
  }
  elsif (($user) = (($cookies->cookie('root')||'') =~ m!^([a-zA-Z]+)$!)) {
    # ...  why is this in an elsif?  :-)
  }

  my $path;

  my $docs = $config->docs_name;

  if ($user) {
    # FIXME|TODO: should expand on ~ instead of using /home
    $user = "/home/$user";
    $path = [
	     (map { "$user/$docs/$_/" } @site_dirs),
	     "$user/$docs/shared/",
	     "$user/$docs/",
	    ];
  }
  else {
    my $root_docs = $config->root_docs;
    $path = [
	     (map { "$root_docs/$_/" } @site_dirs),
	     "$root_docs/shared/",
	     "$root_docs/",
	    ];
  }


  $path = [ $r->document_root ] if $r->dir_config('UseDocumentRoot');
  push @$path, "$root/apache/root_templates/";

  #warn Data::Dumper->Dump([\$path], [qw(path)]);
  
  return $path;

}

sub evaluate_template {
  my $self      = shift;
  my $template  = shift;

  my $tpl_params    = { %{$self->tpl_params }, ($_[0] and ref $_[0] eq 'HASH') ? %{$_[0]} : @_ };

  my $r = $self->r;

  $tpl_params->{r} = $r; 
  $tpl_params->{notes} = $r->pnotes('combust_notes'); 
  $tpl_params->{root} = $root;  # localroot anyone?
  $tpl_params->{siteconfig} = $self->site && $self->config->site->{$self->site};

  $tpl_params->{combust} = $self;

  $tpl_params->{site} = $self->site
    unless $tpl_params->{site};

  my $output = eval { $self->tt->process($template, $tpl_params, { site => $tpl_params->{site} } ) };

  delete $tpl_params->{combust};

  if ($@) {
      warn( (ref $self ? ref $self : $self) . "  - ". $r->uri . ($r->args ? '?' .$r->args : '')
            . " - error processing template $template: $@");
      Ex_ServerError->throw( error => $@ );
  }

  return $output;
}

my $ctemplate;

sub tt {
    $ctemplate ||= Combust::Template->new()
      or die "Could not initialize Combust::Template object: $Template::ERROR";
}

sub provider {
    my $self = shift;
    cluck "combust->provider is deprecated; use combust->tt->provider";
    $self->tt->provider(@_);
}

sub site {
  my $self = shift;
  return $self->{site} if $self->{site};
  return $self->{site} = $self->r->dir_config("site");
}

sub content_type {
  shift->request->content_type(@_);
}

sub send_cached {
  my ($self, $cache, $content_type) = @_;

  $self->r->update_mtime($cache->{created_timestamp});

  $content_type = $cache->{meta_data}->{content_type}
      if $cache->{meta_data}->{content_type};

  return $self->send_output($cache->{data}, $content_type);
}

sub default_character_set {
  'iso-8859-1'
}

sub send_output {
  my $self = shift;
  
  # we used to take $r as the first parameter
  if (ref $_[0] eq "Apache::Request") {
    cluck "send_output doesn't need \$r passed anymore"; 
    shift @_;
  }

  my $output = shift;
  my $content_type = shift || $self->content_type || 'text/html';

  unless (defined $output) {
    cluck "send_output called with undefined output";
    return 404;
  }

  # for some reason mod_perl will sometimes forget to dereference
  # a reference, so let's not try printing those anymore.
  $output = $$output if ref $output and ref $output ne 'GLOB';

  my $r = $self->r;

  $self->cookies->bake_cookies;

  # not that we actually have the /w3c/p3p.xml document
  $self->request->header_out('P3P',qq[CP="NOI DEVo TAIo PSAo PSDo OUR IND UNI NAV", policyref="/w3c/p3p.xml"]);

  my $length;
  if (ref $output eq "GLOB") {
    $length = (stat($output))[7];
  }
  else {
    # eek - this is certainly not correct
    $output = encode_utf8($output) if $content_type =~ m!^text/!;
    $length = length($output);
  }

  $self->request->update_mtime(time) if $r->mtime == 0; 
  
  $r->set_content_length($length);
  $r->set_last_modified();  # set's to whatever update_mtime told us..

  # defining the character set helps in handling the CERT advisory
  # regarding  "cross site scripting vulnerabilities" 
  #   http://www.cert.org/tech_tips/malicious_code_mitigation.html
  #  $content_type .= "; charset=" . $self->default_character_set
  #    if $content_type =~ m/^text/ and $content_type !~ m/charset=/;
  $content_type .= "; charset=utf-8" if $content_type =~ m/^text/ and $content_type !~ m/charset=/;
  $r->content_type($content_type);
  #warn "content_type: $content_type";

  if ((my $rc = $r->meets_conditions) != OK) {
    # this didn't work with just returning $rc -- need to check if it works now.
    $r->status($rc);
    return $rc;
  }

  $self->request->send_http_header;

  #warn Data::Dumper->Dump([\$output], [qw(output)]);

  # if all that is requested is HEAD
  # don't send the body
  return OK if $r->header_only;

  if (ref $output eq "GLOB") {
    print <$output>;  
  }
  else {
    print $output;
  }

  # TODO: need to get the status from further up the chain and return it correctly here.
  return OK;
}

sub redirect {
  my $self = shift;
  my $url = shift;
  my $ref_url = ref $url || '';
  if ($ref_url =~ m/^Apache/) {  # if we got passed an $r as the first parameter
    cluck "You don't need to pass \$r to the redirect method";
    $url = shift;
  }

  my $permanent = shift;

  $self->cookies->bake_cookies;

  $url = $url->abs if ref $url =~ m/^URI/;

  # this should really check for a complete URI or some such; we'll do
  # that when it breaks on a ftp:// or whatever redirect :-)
  unless ($url =~ m!^https?://!i) {
    $url = $config->base_url($self->site) . $url;
  }

  #use Carp qw(cluck);
  #warn "redirecting to [$url]";

  $self->request->header_out('Location' => $url);
  $self->r->status($permanent ? MOVED : REDIRECT);

  my $url_escaped = uri_escape($url);

  my $data = <<EOH;
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<HTML><HEAD><TITLE>Redirect...</TITLE></HEAD><BODY>The document has moved <A HREF="$url_escaped">here</A>.<P></BODY></HTML>
EOH

  $self->request->header_out('Content-Length' => length($data));

  $self->request->send_http_header("text/html");
  print $data;
  return DONE;
}

sub cookies {<commit_msg>Don't _URI_ escape the URL to be put in the redirect HTML<commit_after>package Combust::Control;
use strict;
use Exception::Class ('ControllerException');
use Combust::Constant qw(OK SERVER_ERROR MOVED DONE DECLINED REDIRECT);
use Carp qw(confess cluck carp);
use Digest::SHA1 qw(sha1_hex);
use HTML::Entities ();
use Encode qw(encode_utf8);
use base qw(Combust::Redirect);

# TODO: figure out why we use this; remove it if possible
require bytes;

use Exception::Class ('Ex_ServerError');

use Combust::Cache;
use Combust::Template;
use Combust::Cookies;
use Combust::Secret qw(get_secret);

use Combust::Config;

my $config = Combust::Config->new();

sub config { $config }

my $root = $config->root;

sub r {
  my $self = shift;
  # some day we'll deprecate this - it only works for Apache13 and Apache2
  return $self->request->_r;
}

sub req_param {
  my $self = shift;
  $self->request->req_param(@_);
}

sub param  { cluck "param() deprecated; use tpl_param()"; tpl_param(@_) }
sub params { cluck "params() deprecated; use tpl_params()"; tpl_params(@_) }

sub tpl_param {
  my ($self, $key) = (shift, shift);
  return unless $key;
  #Carp::cluck "param('$key' ...) called" if $key eq "user";
  $self->{params}->{$key} = shift if @_;
  return $self->{params}->{$key};
}

sub tpl_params {
  my $self = shift;
  cluck("tpl_params called with [$self] as self.  Did you configure the handler to call ->handler instead of ->super?")
    unless ref $self;
  cluck('Combust::Control->tpl_params called with parameters, did you mean to call "param"?') if @_;
  $self->{params} || {};
}

sub new {
  my ($class, $r) = @_;

  # return if we are already blessed
  return $class if ref $class;

  my $self = bless( { } , $class);
  
  $self;
}

sub super ($$) {
  my $class   = shift;
  my $r = shift;

  confess(__PACKAGE__ . '->super got called without $r') unless $r;
  return unless $r;

  my $self = $class->new($r);

  Combust::Notes::handler($r);

  $self->tt->set_include_path($self->get_include_path);

  my $status;

  eval {
    $status = OK;
    $status = $self->init if $self->can('init');
  };
  if ($@) {
    cluck "$self->init died: $@";
    return SERVER_ERROR;
  }
  return $status unless $status == OK;

  eval {
      ($status) = $self->handler($self->r);
  };
  cluck "Combust::Control: oops, class handler died with: $@" if $@;
  return SERVER_ERROR if $@;

  # should we do this to make it harder for people to shoot themselves in the foot?
  # $self->_cleanup_params;

  return $status;
}

sub handler {
  my $self = shift;
  unless ($self->can('render')) {
    my $msg = "$self doesn't have a render method; you probably got the inheritance order messed up somewhere.";
    warn $msg;
    die $msg;
  }

  my $redir_status = $self->redirect_check;
  return $redir_status unless $redir_status == DECLINED;

  my ($status, $output, $content_type) = $self->do_request();
  # have to return 'OK' and fake it with r->status or some such to make a custom 404 easily
  return $status unless $status == OK;
  my @r = $self->send_output($output, $content_type);
  if ($self->can('cleanup')) {
      eval { $self->cleanup };
      warn "CLEANUP method failed: $@" if $@; 
  }
  return @r;
}

sub do_request {
  my $self = shift;

  my $cache_info = $self->cache_info || {};

  my ($status, $output, $cache);

  if ($cache_info->{id} 
      && ($cache = Combust::Cache->new( type => ($cache_info->{type} || '') ))
     ) {
    my $cache_data;
    $cache_data = $cache->fetch(id => $cache_info->{id})
      unless $self->req_param('cache_bypass');

    if ($cache_data and $cache_data->{data}) {
      $self->post_process($cache_data->{data});
      $self->r->update_mtime($cache_data->{created_timestamp});
      my ($content_type);
      $content_type = $cache->{meta_data}->{content_type}
	if $cache->{meta_data}->{content_type};
      $status = $cache->{meta_data}->{status}
	if $cache->{meta_data}->{status};

      $status ||= OK;

      return ($status, $cache_data->{data}, $content_type);
    }
  }

  ($status, $output, my $content_type) = eval { $self->render };
  if ($@) {
      warn "render failed: $@";
      $status = SERVER_ERROR;
  }
  return $status unless $status == OK;

  # sometimes we end up here with "OK" but with no content ... gah.
  if ($cache and $output and $status != SERVER_ERROR and !$self->no_cache) {
    $cache_info->{meta_data}->{content_type} = $content_type if $content_type;
    $cache_info->{meta_data}->{status}       = $status || $self->r->status;
    $cache->store( %$cache_info, data => $output );
  }
  
  $status = $self->post_process($output);

  return ($status, $output, $content_type);
}

sub no_cache {
    my $self   = shift;
    my $status = shift;
    $self->{no_cache} = $status if defined $status;
    return $self->{no_cache};
}

sub cache_info {}
sub post_process { OK }

sub _cleanup_params {
  my $self = shift;
  for my $param (keys %{$self->{params}}) {
    delete $self->{params}->{$param};
  }
}

sub get_include_path {
  my $self = shift;

  my $r = $self->r;

  my $site = $self->site;
  unless ($site) {
    my @path = ("$root/apache/root_templates/");
    unshift @path, $r->document_root if $r->dir_config('UseDocumentRoot');
    return \@path;
  }

  my @site_dirs = split /:/, ($config->site->{$site}->{docs_site} || $site);

  #warn Data::Dumper->Dump([\$r], [qw(r)]);

  my $cookies = $self->cookies;

  #warn "param:root: ", $r->param('root');
  #warn "root coookie : ", $cookies->cookie('root');

  my ($user);
  my $root_param = $r->param('root') || '';
  if (($user) = ($root_param =~ m!^/?([a-zA-Z]+)$!)) {
    $cookies->cookie('root', "$user");
  } 
  elsif ($root_param eq "/") {
    # don't set user, reset the cookie
    $cookies->cookie('root', "/");
  }
  elsif (($user) = (($cookies->cookie('root')||'') =~ m!^([a-zA-Z]+)$!)) {
    # ...  why is this in an elsif?  :-)
  }

  my $path;

  my $docs = $config->docs_name;

  if ($user) {
    # FIXME|TODO: should expand on ~ instead of using /home
    $user = "/home/$user";
    $path = [
	     (map { "$user/$docs/$_/" } @site_dirs),
	     "$user/$docs/shared/",
	     "$user/$docs/",
	    ];
  }
  else {
    my $root_docs = $config->root_docs;
    $path = [
	     (map { "$root_docs/$_/" } @site_dirs),
	     "$root_docs/shared/",
	     "$root_docs/",
	    ];
  }


  $path = [ $r->document_root ] if $r->dir_config('UseDocumentRoot');
  push @$path, "$root/apache/root_templates/";

  #warn Data::Dumper->Dump([\$path], [qw(path)]);
  
  return $path;

}

sub evaluate_template {
  my $self      = shift;
  my $template  = shift;

  my $tpl_params    = { %{$self->tpl_params }, ($_[0] and ref $_[0] eq 'HASH') ? %{$_[0]} : @_ };

  my $r = $self->r;

  $tpl_params->{r} = $r; 
  $tpl_params->{notes} = $r->pnotes('combust_notes'); 
  $tpl_params->{root} = $root;  # localroot anyone?
  $tpl_params->{siteconfig} = $self->site && $self->config->site->{$self->site};

  $tpl_params->{combust} = $self;

  $tpl_params->{site} = $self->site
    unless $tpl_params->{site};

  my $output = eval { $self->tt->process($template, $tpl_params, { site => $tpl_params->{site} } ) };

  delete $tpl_params->{combust};

  if ($@) {
      warn( (ref $self ? ref $self : $self) . "  - ". $r->uri . ($r->args ? '?' .$r->args : '')
            . " - error processing template $template: $@");
      Ex_ServerError->throw( error => $@ );
  }

  return $output;
}

my $ctemplate;

sub tt {
    $ctemplate ||= Combust::Template->new()
      or die "Could not initialize Combust::Template object: $Template::ERROR";
}

sub provider {
    my $self = shift;
    cluck "combust->provider is deprecated; use combust->tt->provider";
    $self->tt->provider(@_);
}

sub site {
  my $self = shift;
  return $self->{site} if $self->{site};
  return $self->{site} = $self->r->dir_config("site");
}

sub content_type {
  shift->request->content_type(@_);
}

sub send_cached {
  my ($self, $cache, $content_type) = @_;

  $self->r->update_mtime($cache->{created_timestamp});

  $content_type = $cache->{meta_data}->{content_type}
      if $cache->{meta_data}->{content_type};

  return $self->send_output($cache->{data}, $content_type);
}

sub default_character_set {
  'iso-8859-1'
}

sub send_output {
  my $self = shift;
  
  # we used to take $r as the first parameter
  if (ref $_[0] eq "Apache::Request") {
    cluck "send_output doesn't need \$r passed anymore"; 
    shift @_;
  }

  my $output = shift;
  my $content_type = shift || $self->content_type || 'text/html';

  unless (defined $output) {
    cluck "send_output called with undefined output";
    return 404;
  }

  # for some reason mod_perl will sometimes forget to dereference
  # a reference, so let's not try printing those anymore.
  $output = $$output if ref $output and ref $output ne 'GLOB';

  my $r = $self->r;

  $self->cookies->bake_cookies;

  # not that we actually have the /w3c/p3p.xml document
  $self->request->header_out('P3P',qq[CP="NOI DEVo TAIo PSAo PSDo OUR IND UNI NAV", policyref="/w3c/p3p.xml"]);

  my $length;
  if (ref $output eq "GLOB") {
    $length = (stat($output))[7];
  }
  else {
    # eek - this is certainly not correct
    $output = encode_utf8($output) if $content_type =~ m!^text/!;
    $length = length($output);
  }

  $self->request->update_mtime(time) if $r->mtime == 0; 
  
  $r->set_content_length($length);
  $r->set_last_modified();  # set's to whatever update_mtime told us..

  # defining the character set helps in handling the CERT advisory
  # regarding  "cross site scripting vulnerabilities" 
  #   http://www.cert.org/tech_tips/malicious_code_mitigation.html
  #  $content_type .= "; charset=" . $self->default_character_set
  #    if $content_type =~ m/^text/ and $content_type !~ m/charset=/;
  $content_type .= "; charset=utf-8" if $content_type =~ m/^text/ and $content_type !~ m/charset=/;
  $r->content_type($content_type);
  #warn "content_type: $content_type";

  if ((my $rc = $r->meets_conditions) != OK) {
    # this didn't work with just returning $rc -- need to check if it works now.
    $r->status($rc);
    return $rc;
  }

  $self->request->send_http_header;

  #warn Data::Dumper->Dump([\$output], [qw(output)]);

  # if all that is requested is HEAD
  # don't send the body
  return OK if $r->header_only;

  if (ref $output eq "GLOB") {
    print <$output>;  
  }
  else {
    print $output;
  }

  # TODO: need to get the status from further up the chain and return it correctly here.
  return OK;
}

sub redirect {
  my $self = shift;
  my $url = shift;
  my $ref_url = ref $url || '';
  if ($ref_url =~ m/^Apache/) {  # if we got passed an $r as the first parameter
    cluck "You don't need to pass \$r to the redirect method";
    $url = shift;
  }

  my $permanent = shift;

  $self->cookies->bake_cookies;

  $url = $url->abs if ref $url =~ m/^URI/;

  # this should really check for a complete URI or some such; we'll do
  # that when it breaks on a ftp:// or whatever redirect :-)
  unless ($url =~ m!^https?://!i) {
    $url = $config->base_url($self->site) . $url;
  }

  #use Carp qw(cluck);
  #warn "redirecting to [$url]";

  $self->request->header_out('Location' => $url);
  $self->r->status($permanent ? MOVED : REDIRECT);

  my $url_escaped = HTML::Entities::encode_entities($url);

  my $data = <<EOH;
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<HTML><HEAD><TITLE>Redirect...</TITLE></HEAD><BODY>The document has moved <A HREF="$url_escaped">here</A>.<P></BODY></HTML>
EOH

  $self->request->header_out('Content-Length' => length($data));

  $self->request->send_http_header("text/html");
  print $data;
  return DONE;
}

sub cookies {
{'fraction_non_alphanumeric': 0.18565637398654478, 'fraction_numerical': 0.0027600483008452647, 'mean_word_length': 3.4001897533206833, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 377, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3911787', 'n_tokens_mistral': 8643, 'n_tokens_neox': 8352, 'n_words': 2156}