<commit_before>#!/usr/bin/env bash

# giraffe-wrangler.sh: Run and profile vg gaffe and analyze the results.

set -e

usage() {
    # Print usage to stderr
    exec 1>&2
    printf "Usage: $0 [Options] FASTA XG_INDEX GCSA_INDEX GBWT_INDEX MINIMIZER_INDEX DISTANCE_INDEX SIM_GAM REAL_FASTQ\n"
    printf "\n"
    printf "Inputs may be files or S3 URLs.\n"
    printf "\n"
    printf "Arguments:\n"
    printf "  FASTA            FASTA reference to run bwa-mem against; may be \"\"\n"
    printf "  XG_INDEX         XG to annotate reads with positions, with corresponding .gg GBWTGraph\n"
    printf "  GCSA_INDEX       GCSA (with LCP) for running vg map\n"
    printf "  GBWT_INDEX       Haplotypes for mapping with Giraffe\n"
    printf "  MINIMIZER_INDEX  Minimizers for mapping with Giraffe\n"
    printf "  DISTANCE_INDEX   Distances for mapping with Giraffe\n"
    printf "  SIM_GAM          Simulated reads for measuring mapping accuracy; may be \"\"\n"
    printf "  REAL_FASTQ       Real reads for measuring mapping performance; may be \"\"\n"
    printf "\n"
    printf "Options:\n"
    printf "  -s DEST Save alignments and other internal files to DEST (directory or S3 url)\n"
    printf "  -t N    Use N threads\n"
    printf "\n"
    exit 1
}

# Define where we should save our output
OUTPUT_DEST=""

# Define the thread count for everyone. Can be changed with -t.
# Should fit on a NUMA node
THREAD_COUNT=24

while getopts ":s:t:" o; do
    case "${o}" in
        s)
            OUTPUT_DEST="${OPTARG}"
            ;;
        t)
            THREAD_COUNT="${OPTARG}"
            ;;
        ?)
            usage
            ;;
    esac
done
shift $((OPTIND-1))
if [[ "$#" -lt "8" ]]; then
    # Too few arguments
    usage
fi

echo "Using ${THREAD_COUNT} threads"

fetch_input() {
    # Download the specified file, if not empty and not a file already.
    # Dumps all files into the current directory as their basenames
    # Output the new filename
    if [[ ! -z "${1}" && "${1}" == s3://* ]] ; then
        aws s3 --quiet cp "${1}" "$(basename "${1}")"
        basename "${1}"
    else
        echo "${1}"
    fi
}

FASTA="$(fetch_input "${1}")"
if [[ ! -z ${FASTA} ]] ; then
    # We have a FASTA
    for EXT in amb ann bwt fai pac sa ; do
        # Make sure we have all the indexes adjacent to the FASTA
        fetch_input "${1}.${EXT}" >/dev/null
    done
fi
shift
XG_INDEX="$(fetch_input "${1}")"
# Make sure we have the GBWTGraph pre-made
GBWT_GRAPH="$(fetch_input "${1%.xg}.gg")"
shift
GCSA_INDEX="$(fetch_input "${1}")"
LCP_INDEX="$(fetch_input "${1}.lcp")"
shift
GBWT_INDEX="$(fetch_input "${1}")"
shift
MINIMIZER_INDEX="$(fetch_input "${1}")"
shift
DISTANCE_INDEX="$(fetch_input "${1}")"
shift
SIM_GAM="$(fetch_input "${1}")"
shift
REAL_FASTQ="$(fetch_input "${1}")"
shift

if [ -f "$GBWT_GRAPH" ]; then
    GIRAFFE_GRAPH=(-g "${GBWT_GRAPH}")
else
    GIRAFFE_GRAPH=(-x "${XG_INDEX}")
fi

echo "Indexes:"
echo "${XG_INDEX}"
echo "${GBWT_GRAPH}"
echo "${GCSA_INDEX}"
echo "${LCP_INDEX}"
echo "${GBWT_INDEX}"
echo "${MINIMIZER_INDEX}"
echo "${DISTANCE_INDEX}"
echo "${SIM_GAM}"
echo "${REAL_FASTQ}"

# Define the Giraffe parameters
GIRAFFE_OPTS=(-C 1500 -F 0.8 -e 300 -a 6 -s 50 -u 0.4 -v 1 -w 20 -i)

# Define a work directory
# TODO: this requires GNU mptemp
WORK="$(mktemp -d)"

# Check for NUMA. If we have NUMA and no numactl results may be unreliable
NUMA_COUNT="$(lscpu | grep "NUMA node(s)" | cut -f3- -d' ' | tr -d ' ')"
NUMA_PREFIX=""
NUMA_WARNING=0

if [[ "${NUMA_COUNT}" -gt "1" ]] ; then
    if which numactl >/dev/null 2>&1 ; then
        # Run everything on one NUMA node
        NUMA_PREFIX="numactl --cpunodebind=0 --membind=0"
    else
        # We should warn in the report that NUMA may confound the results
        NUMA_WARNING=1
    fi
fi

if [[ ! -z "${REAL_FASTQ}" ]] ; then
    if which perf >/dev/null 2>&1 ; then
        # Record profile.
        # Do this first because perf is likely to be misconfigured and we want to fail fast.
        <commit_msg>Adopt the default Giraffe settings for filters<commit_after>#!/usr/bin/env bash

# giraffe-wrangler.sh: Run and profile vg gaffe and analyze the results.

set -e

usage() {
    # Print usage to stderr
    exec 1>&2
    printf "Usage: $0 [Options] FASTA XG_INDEX GCSA_INDEX GBWT_INDEX MINIMIZER_INDEX DISTANCE_INDEX SIM_GAM REAL_FASTQ\n"
    printf "\n"
    printf "Inputs may be files or S3 URLs.\n"
    printf "\n"
    printf "Arguments:\n"
    printf "  FASTA            FASTA reference to run bwa-mem against; may be \"\"\n"
    printf "  XG_INDEX         XG to annotate reads with positions, with corresponding .gg GBWTGraph\n"
    printf "  GCSA_INDEX       GCSA (with LCP) for running vg map\n"
    printf "  GBWT_INDEX       Haplotypes for mapping with Giraffe\n"
    printf "  MINIMIZER_INDEX  Minimizers for mapping with Giraffe\n"
    printf "  DISTANCE_INDEX   Distances for mapping with Giraffe\n"
    printf "  SIM_GAM          Simulated reads for measuring mapping accuracy; may be \"\"\n"
    printf "  REAL_FASTQ       Real reads for measuring mapping performance; may be \"\"\n"
    printf "\n"
    printf "Options:\n"
    printf "  -s DEST Save alignments and other internal files to DEST (directory or S3 url)\n"
    printf "  -t N    Use N threads\n"
    printf "\n"
    exit 1
}

# Define where we should save our output
OUTPUT_DEST=""

# Define the thread count for everyone. Can be changed with -t.
# Should fit on a NUMA node
THREAD_COUNT=24

while getopts ":s:t:" o; do
    case "${o}" in
        s)
            OUTPUT_DEST="${OPTARG}"
            ;;
        t)
            THREAD_COUNT="${OPTARG}"
            ;;
        ?)
            usage
            ;;
    esac
done
shift $((OPTIND-1))
if [[ "$#" -lt "8" ]]; then
    # Too few arguments
    usage
fi

echo "Using ${THREAD_COUNT} threads"

fetch_input() {
    # Download the specified file, if not empty and not a file already.
    # Dumps all files into the current directory as their basenames
    # Output the new filename
    if [[ ! -z "${1}" && "${1}" == s3://* ]] ; then
        aws s3 --quiet cp "${1}" "$(basename "${1}")"
        basename "${1}"
    else
        echo "${1}"
    fi
}

FASTA="$(fetch_input "${1}")"
if [[ ! -z ${FASTA} ]] ; then
    # We have a FASTA
    for EXT in amb ann bwt fai pac sa ; do
        # Make sure we have all the indexes adjacent to the FASTA
        fetch_input "${1}.${EXT}" >/dev/null
    done
fi
shift
XG_INDEX="$(fetch_input "${1}")"
# Make sure we have the GBWTGraph pre-made
GBWT_GRAPH="$(fetch_input "${1%.xg}.gg")"
shift
GCSA_INDEX="$(fetch_input "${1}")"
LCP_INDEX="$(fetch_input "${1}.lcp")"
shift
GBWT_INDEX="$(fetch_input "${1}")"
shift
MINIMIZER_INDEX="$(fetch_input "${1}")"
shift
DISTANCE_INDEX="$(fetch_input "${1}")"
shift
SIM_GAM="$(fetch_input "${1}")"
shift
REAL_FASTQ="$(fetch_input "${1}")"
shift

if [ -f "$GBWT_GRAPH" ]; then
    GIRAFFE_GRAPH=(-g "${GBWT_GRAPH}")
else
    GIRAFFE_GRAPH=(-x "${XG_INDEX}")
fi

echo "Indexes:"
echo "${XG_INDEX}"
echo "${GBWT_GRAPH}"
echo "${GCSA_INDEX}"
echo "${LCP_INDEX}"
echo "${GBWT_INDEX}"
echo "${MINIMIZER_INDEX}"
echo "${DISTANCE_INDEX}"
echo "${SIM_GAM}"
echo "${REAL_FASTQ}"

# Define the Giraffe parameters
GIRAFFE_OPTS=(-i)

# Define a work directory
# TODO: this requires GNU mptemp
WORK="$(mktemp -d)"

# Check for NUMA. If we have NUMA and no numactl results may be unreliable
NUMA_COUNT="$(lscpu | grep "NUMA node(s)" | cut -f3- -d' ' | tr -d ' ')"
NUMA_PREFIX=""
NUMA_WARNING=0

if [[ "${NUMA_COUNT}" -gt "1" ]] ; then
    if which numactl >/dev/null 2>&1 ; then
        # Run everything on one NUMA node
        NUMA_PREFIX="numactl --cpunodebind=0 --membind=0"
    else
        # We should warn in the report that NUMA may confound the results
        NUMA_WARNING=1
    fi
fi

if [[ ! -z "${REAL_FASTQ}" ]] ; then
    if which perf >/dev/null 2>&1 ; then
        # Record profile.
        # Do this first because perf is likely to be misconfigured and we want to fail fast.
        
{'fraction_non_alphanumeric': 0.13720580871306962, 'fraction_numerical': 0.011892839258888333, 'mean_word_length': 2.8060981419723676, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '746065', 'n_tokens_mistral': 2880, 'n_tokens_neox': 2766, 'n_words': 936}