<commit_before>#!/usr/bin/perl -w
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# $Source: src/build/trace/tracehash_hb.pl $
#
# OpenPOWER HostBoot Project
#
# COPYRIGHT International Business Machines Corp. 2011,2014
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG
# File tracehash.pl created by <NAME>.
# 05/08/2011 - Update by andrewg to work in host boot environment

use strict;

sub determine_args();
sub launch_cpp_and_parse($$);
sub cpp_dir($);
sub read_string_file();
sub collect_files($);
sub assimilate_file($);
sub hash_strings();
sub write_string_file();
sub help();

select (STDERR);
$| = 1;		# Make all prints to STDERR flush the buffer immediately
select (STDOUT);
$| = 1;		# Make all prints to STDOUT flush the buffer immediately

# Constants
my $HEAD_SEP = "|||";
my $HEAD_EYE_CATCHER = "#FSP_TRACE_v";
my $HEAD_BUILD_FLAG = "BUILD:";
my $HEAD_VER_FLAG = 2;
my $BB_STRING_FILE = "/opt/fsp/etc/BB_StringFile";

# Global Variables
my $debug = 0;
my $seperator = "&&&&";
my $file_name = "trexStringFile";
my $in_sand;
my ($backing) = $ENV{'bb'};
my $hash_prog = "trexhash";    #default to in path
my $build = ""; 
my ($sandbox) = $ENV{'SANDBOX'} || "";
my ($context) = $ENV{'CONTEXT'} || "";
my ($sandboxbase) = $ENV{'SANDBOXBASE'} || "";
my ($bb);
my ($sourcebase) = "$sandboxbase/src";
my ($version) = $HEAD_VER_FLAG; # default to oldest version
my ($collect) = 0;
my ($INCLUDE, $Arg, $file, $dir, $string_file);
my $args = "";

my $fail_on_collision = 0; # 1 = exit with error if hash collision occurs
my $hash_filename_too = 0; # 1 = hash is calculated over format string + filename

print "sourcebase = $sourcebase\n" if $debug;
print "sandbox = $sandbox\n" if $debug;
print "backing = $backing\n" if $debug;
print "context = $context\n" if $debug;

if ($context =~ /x86/)
{
    $bb = "i586-pc-linux-gnu";
}
else
{
    $bb = "powerpc-linux";
}

if(($sourcebase =~ /\w+/) && ($sandbox =~ /\w+/))
{
    $INCLUDE = "-I $sandboxbase/export/$context/fips/include  -I $backing/export/$context/fips/include -I /opt/fsp/$bb/include/fsp -I/opt/fsp/$bb/include/ -include /opt/fsp/$bb/include/fsp/tracinterface.H";
}
else
{
    print "Not in Sandbox so guessing Include Paths...\n" if $debug;
    $INCLUDE = "-I/opt/fsp/i586-pc-linux-gnu/include/fsp -I/opt/fsp/i586-pc-linux-gnu/include/ -include /opt/fsp/i586-pc-linux-gnu/include/fsp/tracinterface.H";
}

# I/P Series work in ODE sandbox env.
if ($sandboxbase =~ /\w+/)
{
    $in_sand = 1;
    print "backing = $backing\n" if $debug;
}
else
{
    $in_sand = 0;
}



#  Parse the input parameters.

while (@ARGV) {
    $Arg = shift;

    if ($Arg eq "-h" || $Arg eq "-H") {
        help();
        exit(127);
    }
    if ($Arg eq "-f") {
        $file = shift;
        next;
    }
    if ($Arg eq "-d") {
        $dir = shift;
        next;
    }
    if ($Arg eq "-s") {
        $string_file = shift;
        next;
    }
    if ($Arg eq "-c") {
        $collect = 1;
        next;
    }
    if ($Arg eq "-v") {
	$debug = 1;
	print "debug on\n" if $debug;
	next;
    }
    if ($Arg eq "-C") { # fail if a hash collision is detected
        $fail_on_collision = 1;
        next;
    }
    if ($Arg eq "-F") { # hash is calculated over format string + file name
        $hash_filename_too = 1;
        next;
    }
    if ($Arg eq "-S") {
        $BB_STRING_FILE = "";
        next;
    }

    #just pass it onto compiler
    $args = $args . " " . $Arg;
}

print "args = $args\n" if $debug;

if (!$file && !$dir && !$in_sand) {
    help();
    exit(127);
}

#################################
#          M A I N              #
#################################

my $clock = `date`;

$build = $HEAD_EYE_CATCHER . "$HEAD_VER_FLAG" . $HEAD_SEP . $clock . $HEAD_SEP  . $HEAD_BUILD_FLAG;

$build =~ s/\n//g;

# Global array to hold the parsed TRAC macro calls.
my @strings = ();

# Assoc. arrays to hold hash|string values.
my %string_file_array = ();
my %hash_strings_array = ();

# Check all provided arguments and look for defaults if not provided by user<commit_msg>Change tracehash_hb.pl to use gmtime<commit_after>#!/usr/bin/perl -w
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# $Source: src/build/trace/tracehash_hb.pl $
#
# OpenPOWER HostBoot Project
#
# Contributors Listed Below - COPYRIGHT 2011,2015
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG
# File tracehash.pl created by <NAME>.
# 05/08/2011 - Update by andrewg to work in host boot environment

use strict;

sub determine_args();
sub launch_cpp_and_parse($$);
sub cpp_dir($);
sub read_string_file();
sub collect_files($);
sub assimilate_file($);
sub hash_strings();
sub write_string_file();
sub help();

select (STDERR);
$| = 1;		# Make all prints to STDERR flush the buffer immediately
select (STDOUT);
$| = 1;		# Make all prints to STDOUT flush the buffer immediately

# Constants
my $HEAD_SEP = "|||";
my $HEAD_EYE_CATCHER = "#FSP_TRACE_v";
my $HEAD_BUILD_FLAG = "BUILD:";
my $HEAD_VER_FLAG = 2;
my $BB_STRING_FILE = "/opt/fsp/etc/BB_StringFile";

# Global Variables
my $debug = 0;
my $seperator = "&&&&";
my $file_name = "trexStringFile";
my $in_sand;
my ($backing) = $ENV{'bb'};
my $hash_prog = "trexhash";    #default to in path
my $build = ""; 
my ($sandbox) = $ENV{'SANDBOX'} || "";
my ($context) = $ENV{'CONTEXT'} || "";
my ($sandboxbase) = $ENV{'SANDBOXBASE'} || "";
my ($bb);
my ($sourcebase) = "$sandboxbase/src";
my ($version) = $HEAD_VER_FLAG; # default to oldest version
my ($collect) = 0;
my ($INCLUDE, $Arg, $file, $dir, $string_file);
my $args = "";

my $fail_on_collision = 0; # 1 = exit with error if hash collision occurs
my $hash_filename_too = 0; # 1 = hash is calculated over format string + filename

print "sourcebase = $sourcebase\n" if $debug;
print "sandbox = $sandbox\n" if $debug;
print "backing = $backing\n" if $debug;
print "context = $context\n" if $debug;

if ($context =~ /x86/)
{
    $bb = "i586-pc-linux-gnu";
}
else
{
    $bb = "powerpc-linux";
}

if(($sourcebase =~ /\w+/) && ($sandbox =~ /\w+/))
{
    $INCLUDE = "-I $sandboxbase/export/$context/fips/include  -I $backing/export/$context/fips/include -I /opt/fsp/$bb/include/fsp -I/opt/fsp/$bb/include/ -include /opt/fsp/$bb/include/fsp/tracinterface.H";
}
else
{
    print "Not in Sandbox so guessing Include Paths...\n" if $debug;
    $INCLUDE = "-I/opt/fsp/i586-pc-linux-gnu/include/fsp -I/opt/fsp/i586-pc-linux-gnu/include/ -include /opt/fsp/i586-pc-linux-gnu/include/fsp/tracinterface.H";
}

# I/P Series work in ODE sandbox env.
if ($sandboxbase =~ /\w+/)
{
    $in_sand = 1;
    print "backing = $backing\n" if $debug;
}
else
{
    $in_sand = 0;
}



#  Parse the input parameters.

while (@ARGV) {
    $Arg = shift;

    if ($Arg eq "-h" || $Arg eq "-H") {
        help();
        exit(127);
    }
    if ($Arg eq "-f") {
        $file = shift;
        next;
    }
    if ($Arg eq "-d") {
        $dir = shift;
        next;
    }
    if ($Arg eq "-s") {
        $string_file = shift;
        next;
    }
    if ($Arg eq "-c") {
        $collect = 1;
        next;
    }
    if ($Arg eq "-v") {
	$debug = 1;
	print "debug on\n" if $debug;
	next;
    }
    if ($Arg eq "-C") { # fail if a hash collision is detected
        $fail_on_collision = 1;
        next;
    }
    if ($Arg eq "-F") { # hash is calculated over format string + file name
        $hash_filename_too = 1;
        next;
    }
    if ($Arg eq "-S") {
        $BB_STRING_FILE = "";
        next;
    }

    #just pass it onto compiler
    $args = $args . " " . $Arg;
}

print "args = $args\n" if $debug;

if (!$file && !$dir && !$in_sand) {
    help();
    exit(127);
}

#################################
#          M A I N              #
#################################

my $clock = gmtime;

$build = $HEAD_EYE_CATCHER . "$HEAD_VER_FLAG" . $HEAD_SEP . $clock . $HEAD_SEP  . $HEAD_BUILD_FLAG;

$build =~ s/\n//g;

# Global array to hold the parsed TRAC macro calls.
my @strings = ();

# Assoc. arrays to hold hash|string values.
my %string_file_array = ();
my %hash_strings_array = ();

# Check all provided arguments and look for defaults if not provided by user
{'fraction_non_alphanumeric': 0.1694048525731694, 'fraction_numerical': 0.011968229790011968, 'mean_word_length': 3.1035714285714286, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2739896', 'n_tokens_mistral': 3418, 'n_tokens_neox': 3248, 'n_words': 986}