<commit_before># UTF8DBI.pm re-implementation by <NAME> http://search.cpan.org/~kudinov/
# originally from: http://dysphoria.net/code/perl-utf8/

package UTF8DBI    ; use base DBI    ;
package UTF8DBI::db; use base DBI::db;
package UTF8DBI::st; use base DBI::st;

sub _utf8_ {
  use Encode;
  if    (ref $_ eq 'ARRAY'){ _utf8_() foreach        @$_  }
  elsif (ref $_ eq 'HASH' ){ _utf8_() foreach values %$_  }
  else                     {         Encode::_utf8_on($_) };
  $_;
};

sub fetch             { return _utf8_ for shift->SUPER::fetch            (@_)  };
sub fetchrow_arrayref { return _utf8_ for shift->SUPER::fetchrow_arrayref(@_)  };
sub fetchrow_hashref  { return _utf8_ for shift->SUPER::fetchrow_hashref (@_)  };
sub fetchall_arrayref { return _utf8_ for shift->SUPER::fetchall_arrayref(@_)  };
sub fetchall_hashref  { return _utf8_ for shift->SUPER::fetchall_hashref (@_)  };
sub fetchcol_arrayref { return _utf8_ for shift->SUPER::fetchcol_arrayref(@_)  };

sub fetchrow_array    {                 @{shift->       fetchrow_arrayref(@_)} };

1;<commit_msg>Imported new version of UTF8DBI.pm from http://dysphoria.net/2006/02/05/utf-8-a-go-go/<commit_after># UTF8DBI.pm re-implementation by <NAME> http://search.cpan.org/~kudinov/
# originally from: http://dysphoria.net/code/perl-utf8/
# And patched again by <NAME>, Jan 2007

use DBI 1.21;
use utf8;
use Encode;

package UTF8DBI; use base DBI;
sub _utf8_ {
  if    (ref $_ eq 'ARRAY') {_utf8_() foreach        @$_ }
  elsif (ref $_ eq 'HASH' ) {_utf8_() foreach values %$_ }
else { 
Encode::_utf8_on($_);
$_ = '⁂malformed-UTF8‼' #die "Malformed utf8 string in database"
if (Encode::is_utf8($_) && ! Encode::is_utf8($_, 1));
};
$_;
};


package UTF8DBI::db; use base DBI::db;

sub selectrow_arrayref { return UTF8DBI::_utf8_ for shift->SUPER::selectrow_arrayref(@_) };
sub selectrow_hashref  { return UTF8DBI::_utf8_ for shift->SUPER::selectrow_hashref (@_) };
sub selectall_arrayref { return UTF8DBI::_utf8_ for shift->SUPER::selectall_arrayref(@_) };
sub selectall_hashref  { return UTF8DBI::_utf8_ for shift->SUPER::selectall_hashref (@_) };
sub selectcol_arrayref { return UTF8DBI::_utf8_ for shift->SUPER::selectcol_arrayref(@_) };

sub selectrow_array    { @{shift->selectrow_arrayref(@_)} };


package UTF8DBI::st; use base DBI::st;

sub fetch              { return UTF8DBI::_utf8_ for shift->SUPER::fetch             (@_)  };

1;
{'fraction_non_alphanumeric': 0.14856429463171036, 'fraction_numerical': 0.025384935497295046, 'mean_word_length': 3.866396761133603, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1946418', 'n_tokens_mistral': 923, 'n_tokens_neox': 889, 'n_words': 195}