<commit_before>#!/bin/bash
# =====================================================================
#  HPCJobConfigurator
#  Copyright (C) 2014 by <NAME> <gnuetzi (at) gmail (dot) com>
#
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
# =====================================================================


logFile="${Job:scriptDir}/endLog.log"
> $logFile


yell() { echo "$0: $*" >&2; }
die() { yell "$*"; exit 111; }
try() { "$@" || die "cannot $*"; }


# Collects all FileInfos in each Process folder
# compares to the old FileInfo from last job
# writes a new output FileInfo for this job
executeFilevalidation(){
    # assemble pipeline status
    PYTHONPATH=${General:configuratorModulePath}
    export PYTHONPATH

    python -m HPCJobConfigurator.jobGenerators.jobGeneratorMPI.generatorToolPipeline.scripts.generateFileValidation  \
        --valFileInfoGlobNew "${Pipeline-PostProcess:validationInfoFilesProcessesGlob}" \
        --valFileInfoGlobOld "${Pipeline-PreProcess:validationInfoFile}" \
        --pipelineSpecs="${Pipeline:pipelineSpecs}" \
        --validateOnlyLastModified=True \
        --statusFolder "${Pipeline-PostProcess:statusFolder}" \
        --output="${Pipeline-PostProcess:validationInfoFile}"  \
        >> $logFile 2>&1
        
    return $?
}

# Validates all files in the job folder (over all processes)
# compares to the old FileInfo from last job
# writes a new output FileInfo for this job
executeFilevalidationAll(){
    # assemble pipeline status
    PYTHONPATH=${General:configuratorModulePath}
    export PYTHONPATH

    python -m HPCJobConfigurator.jobGenerators.jobGeneratorMPI.generatorToolPipeline.scripts.generateFileValidation  \
        --searchDirNew "${Pipeline-PostProcess:validationSearchDir}" \
        --valFileInfoGlobOld "${Pipeline-PreProcess:validationInfoFile}" \
        --pipelineSpecs="${Pipeline:pipelineSpecs}" \
        --validateOnlyLastModified=True \
        --statusFolder "${Pipeline-PostProcess:statusFolder}" \
        --output="${Pipeline-PostProcess:validationInfoFile}"  \
        >> $logFile 2>&1
        
    return $?
}

# combine FileInfos
# try executeFilevalidation

# manually validate all files (use this function if executeFilevalidation failed)
try executeFilevalidationAll



# Send email
if [[ "${Cluster:mailAddress}" != "" ]]; then<commit_msg>render fix<commit_after>#!/bin/bash
# =====================================================================
#  HPCJobConfigurator
#  Copyright (C) 2014 by <NAME> <gnuetzi (at) gmail (dot) com>
#
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
# =====================================================================


logFile="${Job:scriptDir}/endLog.log"
> $logFile


yell() { echo "$0: $*" >&2; }
die() { yell "$*"; exit 111; }
try() { "$@" || die "cannot $*"; }


# Collects all FileInfos in each Process folder
# compares to the old FileInfo from last job
# writes a new output FileInfo for this job
executeFilevalidation(){
    # assemble pipeline status
    PYTHONPATH=${General:configuratorModulePath}
    export PYTHONPATH

    python -m HPCJobConfigurator.jobGenerators.jobGeneratorMPI.generatorToolPipeline.scripts.generateFileValidation  \
        --valFileInfoGlobNew "${Pipeline-PostProcess:validationInfoFilesProcessesGlob}" \
        --valFileInfoGlobOld "${Pipeline-PreProcess:validationInfoFile}" \
        --pipelineSpecs="${Pipeline:pipelineSpecs}" \
        --validateOnlyLastModified=True \
        --statusFolder "${Pipeline-PostProcess:statusFolder}" \
        --output="${Pipeline-PostProcess:validationInfoFile}"  \
        >> $logFile 2>&1
        
    return $?
}

# Validates all files in the job folder (over all processes)
# compares to the old FileInfo from last job
# writes a new output FileInfo for this job
executeFilevalidationAll(){
    # assemble pipeline status
    PYTHONPATH=${General:configuratorModulePath}
    export PYTHONPATH

    python -m HPCJobConfigurator.jobGenerators.jobGeneratorMPI.generatorToolPipeline.scripts.generateFileValidation  \
        --searchDirNew "${Pipeline-PostProcess:validationSearchDir}" \
        --valFileInfoGlobOld "${Pipeline-PreProcess:validationInfoFile}" \
        --pipelineSpecs="${Pipeline:pipelineSpecs}" \
        --validateOnlyLastModified=True \
        --statusFolder "${Pipeline-PostProcess:statusFolder}" \
        --output="${Pipeline-PostProcess:validationInfoFile}"  \
        >> $logFile 2>&1
        
    return $?
}

# combine FileInfos
try executeFilevalidation

# manually validate all files (use this function if executeFilevalidation failed)
#try executeFilevalidationAll



# Send email
if [[ "${Cluster:mailAddress}" != "" ]]; then
{'fraction_non_alphanumeric': 0.16593009240658899, 'fraction_numerical': 0.006830052229811169, 'mean_word_length': 4.014098690835851, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114182', 'n_tokens_mistral': 1420, 'n_tokens_neox': 1349, 'n_words': 348}