<commit_before>#!/bin/bash

# Compares two metadata files looking for differences in the included entities
# An XSLT transformation (mdreport.xslt) is used to make an fingerprint of the two metadata
# These finger prints are then compared textually with the diff

# Note that the number of XML elements and XML attributes in an entity is part of the fingerprint that is used in the
# diff. This is a quick and dirty way to compare metadata files that were generated by a similar toolchain.

PROGNAME=`basename $0`

XSLTPROC=`which xsltproc`
DIFF=`which diff`
MDREPORT_XSLT=`dirname $0`/mdreport.xslt

function error_exit
{
    echo "$PROGNAME: $1" 1>&2

    if [ -f "$tmp1" ]; then
        rm $tmp1
    fi
    if [ -f "$tmp2" ]; then
        rm $tmp2
    fi

    exit 2
}

if [ -z "${XSLTPROC}" -o -z "${DIFF}" ]; then
    error_exit "ERROR: Can't find xsltproc or diff"
fi
if [ -z "${MDREPORT_XSLT}" ]; then
    error_exit "ERROR: Can't find '${MDREPORT_XSLT}'"
fi

if [ $# -lt 2 ]; then
    echo "Expected two arguments"
    echo "Usage: ${PROGNAME} <metadata file 1> <metadata file 2>\n"
    echo "Quick and dirty way to check two SAML 2.0 metadata files for differences"
    echo "Returns 0 when files are probably the same, returns 1 when differences are directed"
    echo "Returns 2 on error"
    exit 1
fi

if [ ! -f "$1" ]; then
    error_exit "File '$1' does not exist"
fi
if [ ! -f "$2" ]; then
    error_exit "File '$2' does not exist"
fi

# Create two temp files
tmp1=`mktemp mdcompare.XXXX` && tmp2=`mktemp mdcompare.XXXX`
if [ $? -ne "0" ]; then
    error_exit "Error creating temp files"
fi

# Process them
${XSLTPROC} --nonet ${MDREPORT_XSLT} $1 > ${tmp1} && $XSLTPROC --nonet ${MDREPORT_XSLT} $2 > ${tmp2}
if [ $? -ne "0" ]; then
    error_exit "Error transforming inputs"
fi

# Diff the output
${DIFF} ${tmp1} ${tmp2} | head -n 30
res=$?
if [ $res -gt "1" ]; then
    error_exit "Diff failed"
fi
if [ $res -eq "1" ]; then
    echo "=========="
    echo "'$1' and '$2' differ. Showing first 30 lines:"
    echo $output | head 30
fi

rm ${tmp1}
rm ${tmp2}

exit $res<commit_msg>Fix: always returning 0<commit_after>#!/bin/bash

# Compares two metadata files looking for differences in the included entities
# An XSLT transformation (mdreport.xslt) is used to make an fingerprint of the two metadata
# These finger prints are then compared textually with the diff

# Note that the number of XML elements and XML attributes in an entity is part of the fingerprint that is used in the
# diff. This is a quick and dirty way to compare metadata files that were generated by a similar toolchain.

PROGNAME=`basename $0`

XSLTPROC=`which xsltproc`
DIFF=`which diff`
MDREPORT_XSLT=`dirname $0`/mdreport.xslt

function error_exit
{
    echo "$PROGNAME: $1" 1>&2

    if [ -f "$tmp1" ]; then
        rm $tmp1
    fi
    if [ -f "$tmp2" ]; then
        rm $tmp2
    fi

    exit 2
}

if [ -z "${XSLTPROC}" -o -z "${DIFF}" ]; then
    error_exit "ERROR: Can't find xsltproc or diff"
fi
if [ -z "${MDREPORT_XSLT}" ]; then
    error_exit "ERROR: Can't find '${MDREPORT_XSLT}'"
fi

if [ $# -lt 2 ]; then
    echo "Expected two arguments"
    echo "Usage: ${PROGNAME} <metadata file 1> <metadata file 2>\n"
    echo "Quick and dirty way to check two SAML 2.0 metadata files for differences"
    echo "Returns 0 when files are probably the same, returns 1 when differences are directed"
    echo "Returns 2 on error"
    exit 1
fi

if [ ! -f "$1" ]; then
    error_exit "File '$1' does not exist"
fi
if [ ! -f "$2" ]; then
    error_exit "File '$2' does not exist"
fi

# Create two temp files
tmp1=`mktemp mdcompare.XXXX` && tmp2=`mktemp mdcompare.XXXX`
if [ $? -ne "0" ]; then
    error_exit "Error creating temp files"
fi

# Process them
${XSLTPROC} --nonet ${MDREPORT_XSLT} $1 > ${tmp1} && $XSLTPROC --nonet ${MDREPORT_XSLT} $2 > ${tmp2}
if [ $? -ne "0" ]; then
    error_exit "Error transforming inputs"
fi

# Diff the output
output=`${DIFF} ${tmp1} ${tmp2}`
res=$?
if [ $res -gt "1" ]; then
    error_exit "Diff failed"
fi
if [ $res -eq "1" ]; then
    echo "=========="
    echo "'$1' and '$2' differ. Showing first 30 lines:"
    echo "$output" | head -n 30
fi

rm ${tmp1}
rm ${tmp2}

exit $res
{'fraction_non_alphanumeric': 0.12925493930016663, 'fraction_numerical': 0.021185432039990477, 'mean_word_length': 3.3274974253347063, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '723122', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1445, 'n_words': 586}