<commit_before>#** @file Perl.pm
# @verbatim
#####################################################################
# This program is not guaranteed to work at all, and by using this  #
# program you release the author of any and all liability.          #
#                                                                   #
# You may use this code as long as you are in compliance with the   #
# license (see the LICENSE file) and this notice, disclaimer and    #
# comment box remain intact and unchanged.                          #
#                                                                   #
# Package:     Doxygen::Filter                                      #
# Class:       Perl                                                 #
# Description: Methods for prefiltering Perl code for Doxygen       #
#                                                                   #
# Written by:  <NAME> (jordan at open1x littledot org)         #
# Created:     2011-10-13                                           #
##################################################################### 
# @endverbatim
#
# @copy 2011, <NAME> (<EMAIL>, <EMAIL>)
#*
package Doxygen::Filter::Perl;

use 5.8.8;
use strict;
use warnings;
use parent qw(Doxygen::Filter);
use Log::Log4perl;
use Pod::POM;
use IO::Handle;
use Doxygen::Filter::Perl::POD;
use File::Slurp;
use PPR;

our $VERSION     = '1.73';
$VERSION = eval $VERSION;


# Define State Engine Values
my $hValidStates = {
    'NORMAL'            => 0,
    'COMMENT'           => 1,
    'DOXYGEN'           => 2,
    'POD'               => 3,
    'METHOD'            => 4,
    'DOXYFILE'          => 21,
    'DOXYCLASS'         => 22,
    'DOXYFUNCTION'      => 23,
    'DOXYMETHOD'        => 24,
    'DOXYCOMMENT'       => 25,
};


our %SYSTEM_PACKAGES = map({ $_ => 1 } qw(
    base
    warnings
    strict
    Exporter
    vars
));



sub new
{
    #** @method private new ()
    # This is the constructor and it calls _init() to initiate
    # the various variables
    #*
    my $pkg = shift;
    my $class = ref($pkg) || $pkg;
    
    my $self = {};
    bless ($self, $class);

    # Lets send any passed in arguments to the _init method
    $self->_init(@_);
    return $self;
}

sub DESTROY
{
    #** @method private DESTROY ()
    # This is the destructor
    #*
    my $self = shift;
    $self = {};
}

sub RESETSUB
{
    my $self = shift;
    $self->{'_iOpenBrace'}          = 0;
    $self->{'_iCloseBrace'}         = 0;
    $self->{'_sCurrentMethodName'}  = undef;
    $self->{'_sCurrentMethodType'}  = undef;
    $self->{'_sCurrentMethodState'} = undef;
}

sub RESETFILE  { my $self = shift; $self->{'_aRawFileData'}   = []; $self->{'_aUncommentFileData'}   = [];   }

sub RESETCLASS 
{ 
    my $self = shift;
    #$self->{'_sCurrentClass'}  = 'main'; 
    #push (@{$self->{'_hData'}->{'class'}->{'classorder'}}, 'main');   
    $self->_SwitchClass('main');
}

sub RESETDOXY  { shift->{'_aDoxygenBlock'}  = [];    }
sub RESETPOD   { shift->{'_aPodBlock'}      = [];    }



sub _init
{
    #** @method private _init ()
    # This method is used in the constructor to initiate 
    # the various variables in the object
    #*
    my $self = shift;
    $self->{'_iDebug'}          = 0;
    $self->{'_sState'}          = undef;
    $self->{'_sPreviousState'}  = [];
    $self->_ChangeState('NORMAL');
    $self->{'_hData'}           = {};
    $self->RESETFILE();
    $self->RESETCLASS();
    $self->RESETSUB();
    $self->RESETDOXY();
    $self->RESETPOD();
}




# ----------------------------------------
# Public Methods
# ----------------------------------------
sub GetCurrentClass
{
    my $self = shift;
    return $self->{'_hData'}->{'class'}->{$self->{'_sCurrentClass'}};
}

sub ReadFile 
{
    #** @method public ReadFile ($sFilename)
    # This method will read the contents of the file in to an array
    # and store that in the object as $self->{'_aRawFileData'}
    # @param sFilename - required string (filename to use)
    #*
    my $self = shift;
    my $sFilename = shift;
    my $logger = $self->GetLogger($self);
    $logger->debug("### Entering ReadFile ###");
    
    # Lets record the file name in the data structure
    $self->{'_hData'}->{'filename'}->{'fullpath'} = $sFilename;

    # Replace forward slash with a black slash
    $sFilename =~ s/\\/\//g;
    # Remove windows style drive letters
    $sFilename =~ s/^.*://;
 
    # Lets grab just the file name not the full path for the short name
    $sFilename =~ /^(.*\/)*(.*)$/;
    $self->{'_hData'}->{'filename'}->{'shortname'} = $2;
 
    my $aFileData = read_file($sFilename);
    $aFileData =~ s/\r$//g;
    my @aRawFileData= split /(?<=\n)/, $aFileData;
    $self->{'_aRawFileData'} = \@aRawFileData;
    $self->{'_decommentOK'} = 1;
    eval {
      my $aUncommentFileData_tmp = PPR::decomment2(join($", $aFileData));
      if (defined($PPR::ERROR))
      {
        $self->{'_decommentOK'} = 0;
        $self->{'_aUncommentFileData'} = \@aRawFileData;
      }
      else
      {
        my @aUncommentFileData;
        if (defined($aUncommentFileData_tmp))
        {
          @aUncommentFileData = split /(?<=\n)/, $aUncommentFileData_tmp;
          $self->{'_aUncommentFileData'} = \@aUncommentFileData;
        }
        else
        {
          $self->{'_decommentOK'} = 0;
          $self->{'_aUncommentFileData'} = \@aRawFileData;
        }
      }
    };
    if ($@) {
      $self->{'_decommentOK'} = 0;
      $self->{'_aUncommentFileData'} = \@aRawFileData;<commit_msg>Small incompatibility between PPR version 26 and Perl.pm<commit_after>#** @file Perl.pm
# @verbatim
#####################################################################
# This program is not guaranteed to work at all, and by using this  #
# program you release the author of any and all liability.          #
#                                                                   #
# You may use this code as long as you are in compliance with the   #
# license (see the LICENSE file) and this notice, disclaimer and    #
# comment box remain intact and unchanged.                          #
#                                                                   #
# Package:     Doxygen::Filter                                      #
# Class:       Perl                                                 #
# Description: Methods for prefiltering Perl code for Doxygen       #
#                                                                   #
# Written by:  <NAME> (jordan at open1x littledot org)         #
# Created:     2011-10-13                                           #
##################################################################### 
# @endverbatim
#
# @copy 2011, <NAME> (<EMAIL>, <EMAIL>)
#*
package Doxygen::Filter::Perl;

use 5.8.8;
use strict;
use warnings;
use parent qw(Doxygen::Filter);
use Log::Log4perl;
use Pod::POM;
use IO::Handle;
use Doxygen::Filter::Perl::POD;
use File::Slurp;
use PPR;

our $VERSION     = '1.73';
$VERSION = eval $VERSION;


# Define State Engine Values
my $hValidStates = {
    'NORMAL'            => 0,
    'COMMENT'           => 1,
    'DOXYGEN'           => 2,
    'POD'               => 3,
    'METHOD'            => 4,
    'DOXYFILE'          => 21,
    'DOXYCLASS'         => 22,
    'DOXYFUNCTION'      => 23,
    'DOXYMETHOD'        => 24,
    'DOXYCOMMENT'       => 25,
};


our %SYSTEM_PACKAGES = map({ $_ => 1 } qw(
    base
    warnings
    strict
    Exporter
    vars
));



sub new
{
    #** @method private new ()
    # This is the constructor and it calls _init() to initiate
    # the various variables
    #*
    my $pkg = shift;
    my $class = ref($pkg) || $pkg;
    
    my $self = {};
    bless ($self, $class);

    # Lets send any passed in arguments to the _init method
    $self->_init(@_);
    return $self;
}

sub DESTROY
{
    #** @method private DESTROY ()
    # This is the destructor
    #*
    my $self = shift;
    $self = {};
}

sub RESETSUB
{
    my $self = shift;
    $self->{'_iOpenBrace'}          = 0;
    $self->{'_iCloseBrace'}         = 0;
    $self->{'_sCurrentMethodName'}  = undef;
    $self->{'_sCurrentMethodType'}  = undef;
    $self->{'_sCurrentMethodState'} = undef;
}

sub RESETFILE  { my $self = shift; $self->{'_aRawFileData'}   = []; $self->{'_aUncommentFileData'}   = [];   }

sub RESETCLASS 
{ 
    my $self = shift;
    #$self->{'_sCurrentClass'}  = 'main'; 
    #push (@{$self->{'_hData'}->{'class'}->{'classorder'}}, 'main');   
    $self->_SwitchClass('main');
}

sub RESETDOXY  { shift->{'_aDoxygenBlock'}  = [];    }
sub RESETPOD   { shift->{'_aPodBlock'}      = [];    }



sub _init
{
    #** @method private _init ()
    # This method is used in the constructor to initiate 
    # the various variables in the object
    #*
    my $self = shift;
    $self->{'_iDebug'}          = 0;
    $self->{'_sState'}          = undef;
    $self->{'_sPreviousState'}  = [];
    $self->_ChangeState('NORMAL');
    $self->{'_hData'}           = {};
    $self->RESETFILE();
    $self->RESETCLASS();
    $self->RESETSUB();
    $self->RESETDOXY();
    $self->RESETPOD();
}




# ----------------------------------------
# Public Methods
# ----------------------------------------
sub GetCurrentClass
{
    my $self = shift;
    return $self->{'_hData'}->{'class'}->{$self->{'_sCurrentClass'}};
}

sub ReadFile 
{
    #** @method public ReadFile ($sFilename)
    # This method will read the contents of the file in to an array
    # and store that in the object as $self->{'_aRawFileData'}
    # @param sFilename - required string (filename to use)
    #*
    my $self = shift;
    my $sFilename = shift;
    my $logger = $self->GetLogger($self);
    $logger->debug("### Entering ReadFile ###");
    
    # Lets record the file name in the data structure
    $self->{'_hData'}->{'filename'}->{'fullpath'} = $sFilename;

    # Replace forward slash with a black slash
    $sFilename =~ s/\\/\//g;
    # Remove windows style drive letters
    $sFilename =~ s/^.*://;
 
    # Lets grab just the file name not the full path for the short name
    $sFilename =~ /^(.*\/)*(.*)$/;
    $self->{'_hData'}->{'filename'}->{'shortname'} = $2;
 
    my $aFileData = read_file($sFilename);
    $aFileData =~ s/\r$//g;
    my @aRawFileData= split /(?<=\n)/, $aFileData;
    $self->{'_aRawFileData'} = \@aRawFileData;
    $self->{'_decommentOK'} = 1;
    my $ppr_version;
    $ppr_version = eval $PPR::VERSION;
    eval {
      my $aUncommentFileData_tmp;
      # comparing floating point numbers is a bit nasty, this will do here
      if ($ppr_version < 0.0000259)
      {
        $aUncommentFileData_tmp = PPR::decomment2(join($", $aFileData));
      }
      else
      {
        $aUncommentFileData_tmp = PPR::decomment(join($", $aFileData));
      }
      if (defined($PPR::ERROR))
      {
        $self->{'_decommentOK'} = 0;
        $self->{'_aUncommentFileData'} = \@aRawFileData;
      }
      else
      {
        my @aUncommentFileData;
        if (defined($aUncommentFileData_tmp))
        {
          @aUncommentFileData = split /(?<=\n)/, $aUncommentFileData_tmp;
          $self->{'_aUncommentFileData'} = \@aUncommentFileData;
        }
        else
        {
          $self->{'_decommentOK'} = 0;
          $self->{'_aUncommentFileData'} = \@aRawFileData;
        }
      }
    };
    if ($@) {
      $self->{'_decommentOK'} = 0;
      $self->{'_aUncommentFileData'} = \@aRawFileData;
{'fraction_non_alphanumeric': 0.19026315789473683, 'fraction_numerical': 0.008771929824561403, 'mean_word_length': 1.9589929924733973, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 127, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4907787', 'n_tokens_mistral': 3743, 'n_tokens_neox': 3558, 'n_words': 881}