<commit_before>#!/bin/bash

#################
# Configuration #
#################

# Automatically acceptable libraries
WHITELIST=(libLLVM-3.1svn.so libclang.so) # prefer shared objects libraries to static libraries
BLACKLIST=(liblldb.so) # evict lldb (it contains almost all symbols declared in other libraries)

#####################
# Utility functions #
#####################

# Extracts resources (sub-scripts) located at the end of the script
extract_file() {
    awk -- "
BEGIN { out=0 }
out==1 && /^# \]\]>$/ { out=0 }
out==1 { print }
/^# <!\[$1\[$/ { out=1 }
" "$0"
}

# Extract a few sed expressions
sedUndefSymbols="$(extract_file "sed:extract undefined symbols")"
sedGccStdString="$(extract_file "sed:gcc std::string")"

# Checks if the first argument is listed in the following arguments
# In order to check for a value being included in an array, use:
#   included "value" "${array[@]}"
# and check the return value.
included() {
    what="$1"
    while [ "$#" -gt 0 ]; do
        shift
        [ "$what" == "$1" ] && return 0
    done
    return 1
}

# Like included(), but tests if an ending of the first argument match the proposed arguments
ending_included() {
    what="$1"
    while [ "$#" -gt 0 ]; do
        shift
        [ "${what/%$1/}" != "$what" ] && return 0
    done
    return 1
}

###################
# Begin of script #
###################

# Get our directory so we can call other sibling scripts
progdir="$(dirname "$0")"

# Create a temporary output file and a fifo
tmp="$(mktemp tmp-resolve-compile-XXXXXX.out)"
tmpfifo="$(mktemp tmp-resolve-compile-XXXXXX.fifo)"
rm "$tmpfifo"
mkfifo "$tmpfifo"

# Execute the provided command, and if it fails,
# analyze its output
LDFLAGS=()
if "$@" 2> "$tmp"; then
    newflags="unnecessary"
else
    newflags="none"
    echo "Command failed, finding libs..."
    # Extract undefined symbols
    sed -i -nre "$sedUndefSymbols" -- "$tmp"
    # Simplify g++ output, as it expands some STL template types
    sed -i -re "$sedGccStdString" -- "$tmp"
    lineCount="$(wc -l "$tmp" | sed -re 's/^\s*([0-9]+)\s+.*$/\1/')"
    currLine=0<commit_msg>mktemp(1) backward compatibility: put Xs at the end<commit_after>#!/bin/bash

#################
# Configuration #
#################

# Automatically acceptable libraries
WHITELIST=(libLLVM-3.1svn.so libclang.so) # prefer shared objects libraries to static libraries
BLACKLIST=(liblldb.so) # evict lldb (it contains almost all symbols declared in other libraries)

#####################
# Utility functions #
#####################

# Extracts resources (sub-scripts) located at the end of the script
extract_file() {
    awk -- "
BEGIN { out=0 }
out==1 && /^# \]\]>$/ { out=0 }
out==1 { print }
/^# <!\[$1\[$/ { out=1 }
" "$0"
}

# Extract a few sed expressions
sedUndefSymbols="$(extract_file "sed:extract undefined symbols")"
sedGccStdString="$(extract_file "sed:gcc std::string")"

# Checks if the first argument is listed in the following arguments
# In order to check for a value being included in an array, use:
#   included "value" "${array[@]}"
# and check the return value.
included() {
    what="$1"
    while [ "$#" -gt 0 ]; do
        shift
        [ "$what" == "$1" ] && return 0
    done
    return 1
}

# Like included(), but tests if an ending of the first argument match the proposed arguments
ending_included() {
    what="$1"
    while [ "$#" -gt 0 ]; do
        shift
        [ "${what/%$1/}" != "$what" ] && return 0
    done
    return 1
}

###################
# Begin of script #
###################

# Get our directory so we can call other sibling scripts
progdir="$(dirname "$0")"

# Create a temporary output file and a fifo
tmp="$(mktemp tmp-resolve-compile.out.XXXXXX)"
tmpfifo="$(mktemp tmp-resolve-compile.fifo.XXXXXX)"
rm "$tmpfifo"
mkfifo "$tmpfifo"

# Execute the provided command, and if it fails,
# analyze its output
LDFLAGS=()
if "$@" 2> "$tmp"; then
    newflags="unnecessary"
else
    newflags="none"
    echo "Command failed, finding libs..."
    # Extract undefined symbols
    sed -i -nre "$sedUndefSymbols" -- "$tmp"
    # Simplify g++ output, as it expands some STL template types
    sed -i -re "$sedGccStdString" -- "$tmp"
    lineCount="$(wc -l "$tmp" | sed -re 's/^\s*([0-9]+)\s+.*$/\1/')"
    currLine=0
{'fraction_non_alphanumeric': 0.1994842944210033, 'fraction_numerical': 0.011954992967651195, 'mean_word_length': 3.6380434782608697, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7568769', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1343, 'n_words': 448}