<commit_before>#!/bin/perl

##
## This script cross-references the API functions defined in the
## OpenSSL .num files with the XML idltool source file to determine
## which functions are new and not defined in the AmiSSL API. For the
## new functions, it then attempts to cross-reference those in with
## OpenSSL includes and converts matching function prototypes into XML
## for manual insertion do the end of the amissl.xml file
##

$cc="gcc:bin/gcc-4.0.4";
$cppflags="-mcrt=clib2 -I../include";
@num_files=("../openssl/util/libssl.num", "../openssl/util/libcrypto.num");
$idl_file="../include/xml/amissl.xml";
$includes="../openssl/include/openssl";
@includes_ignore=("safestack.h","asn1_mac.h");
$tmp_dir="newXML";

#######################################################################

use constant DONE_NONE => 0x00;
use constant DONE_RAW => 0x01;
use constant DONE_PREPROCESSED => 0x02;

# remove leading & trailing whitespace
sub trim {
   my $s = shift;
   $s =~ s/^\s+|\s+$//g;
   return $s
};

# read existing .xml file
sub get_idl {
   open(my $fh, "<", @_) || die "Could not open file '@_' $!";
   $idl = do { local $/; <$fh> };
   close $fh;
}

# check symbols in .num files and cross-reference with existing .xml
sub get_new_symbols {
   open(my $fh, "<", @_) || die "Could not open file '@_' $!";

   while (my $row = <$fh>) {
      my @columns = split /\s+/, $row;
      if($columns[3] =~ /^EXIST:\!.*:FUNCTION:.*/ ||
         $columns[3] =~ /^EXIST:EXPORT_VAR_AS_FUNCTION.*:FUNCTION:.*/ ||
         $columns[3] =~ /^EXIST::FUNCTION:.*/)
      {
         if ($idl !~ /$columns[0]/ ) {
            my $unimplemented = 0;
            if ($columns[3] =~ /(STDIO|KRB5|JPAKE|SCTP|EC_NISTP_64_GCC_128)/) {
               $unimplemented = 1;
            }
            push @symbols, { name => $columns[0], unimplemented => $unimplemented, xml => "", done => DONE_NONE };
         }
      }
   }

   close $fh;
}

# get list of header files
sub get_headers {
   my @files = <@_/*.h>;
   @headers = ();
   foreach my $file (@files) {
      my $use = 1;
      foreach my $ignore (@includes_ignore) {
         if ($file =~ /$ignore$/ ) {
            $use = 0;
         }
      }
      if ($use) {
         push @headers, $file;<commit_msg>fixed symbol XML lookup from incorrectly making partial matches<commit_after>#!/bin/perl

##
## This script cross-references the API functions defined in the
## OpenSSL .num files with the XML idltool source file to determine
## which functions are new and not defined in the AmiSSL API. For the
## new functions, it then attempts to cross-reference those in with
## OpenSSL includes and converts matching function prototypes into XML
## for manual insertion do the end of the amissl.xml file
##

$cc="gcc:bin/gcc-4.0.4";
$cppflags="-mcrt=clib2 -I../include";
@num_files=("../openssl/util/libssl.num", "../openssl/util/libcrypto.num");
$idl_file="../include/xml/amissl.xml";
$includes="../openssl/include/openssl";
@includes_ignore=("safestack.h","asn1_mac.h");
$tmp_dir="newXML";

#######################################################################

use constant DONE_NONE => 0x00;
use constant DONE_RAW => 0x01;
use constant DONE_PREPROCESSED => 0x02;

# remove leading & trailing whitespace
sub trim {
   my $s = shift;
   $s =~ s/^\s+|\s+$//g;
   return $s
};

# read existing .xml file
sub get_idl {
   open(my $fh, "<", @_) || die "Could not open file '@_' $!";
   $idl = do { local $/; <$fh> };
   close $fh;
}

# check symbols in .num files and cross-reference with existing .xml
sub get_new_symbols {
   open(my $fh, "<", @_) || die "Could not open file '@_' $!";

   while (my $row = <$fh>) {
      my @columns = split /\s+/, $row;
      if($columns[3] =~ /^EXIST:\!.*:FUNCTION:.*/ ||
         $columns[3] =~ /^EXIST:EXPORT_VAR_AS_FUNCTION.*:FUNCTION:.*/ ||
         $columns[3] =~ /^EXIST::FUNCTION:.*/)
      {
         if ($idl !~ /name="$columns[0]"/ ) {
            my $unimplemented = 0;
            if ($columns[3] =~ /(STDIO|KRB5|JPAKE|SCTP|EC_NISTP_64_GCC_128)/) {
               $unimplemented = 1;
            }
            push @symbols, { name => $columns[0], unimplemented => $unimplemented, xml => "", done => DONE_NONE };
         }
      }
   }

   close $fh;
}

# get list of header files
sub get_headers {
   my @files = <@_/*.h>;
   @headers = ();
   foreach my $file (@files) {
      my $use = 1;
      foreach my $ignore (@includes_ignore) {
         if ($file =~ /$ignore$/ ) {
            $use = 0;
         }
      }
      if ($use) {
         push @headers, $file;
{'fraction_non_alphanumeric': 0.2010547132498352, 'fraction_numerical': 0.013183915622940013, 'mean_word_length': 2.815590947191953, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '621911', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1547, 'n_words': 415}