<commit_before>#!/usr/bin/env bash
if [ "x$BASH" = x ] || [ ! "$BASH_VERSINFO" ] || [ "$BASH_VERSINFO" -lt 4 ]; then
  echo "Error: Must use bash version 4+." >&2
  exit 1
fi
set -ue

Usage="Usage: \$ $(basename "$0") [options] command [command args]
Measure the resources used by a command and all of its children.
Prints 8 tab-delimited columns:
0. The -i argument, if given.
1. ps %CPU: The maximum percentage of CPU used at any one time.
2. ps %MEM: The maximum percentage of RAM used at any one time.
3. ps  VSZ: The maximum virtual memory size (in KB) used at any one time.
4. ps  RSS: The maximum resident set size (in KB) used at any one time.
5. time -f %e: Total number of elapsed wall clock seconds.
6. time -f %S: Total CPU-seconds used by the kernel on behalf of the command.
7. time -f %U: Total CPU-seconds of user-mode execution time.
8. time -f %M: Maximum resident set size in KB.
-i: A string that will be prepended to the output as the first column.
-o: Output file. The command's stdout will be piped here. Default: /dev/null
-l: Log file. The command's stderr will be piped here. Default: /dev/null
-s: Run under slurm. Prefixes the command with \"srun\".
-S: Arguments to give the slurm \"srun\" command.
-D: Turn on debug mode."

function main {

  # Get arguments.
  id=
  outfile=/dev/null
  logfile=/dev/null
  slurm=
  slurm_args=
  debug=
  while getopts "i:o:l:sS:Dh" opt; do
    case "$opt" in
      i) id="$OPTARG";;
      o) outfile="$OPTARG";;
      l) logfile="$OPTARG";;
      s) slurm=true;;
      S) slurm_args="$OPTARG";;
      D) debug='-D';;
      [h?]) fail "$Usage";;
    esac
  done
  command="${@:$OPTIND:1}"
  command_args="${@:$OPTIND+1}"

  if ! [[ "$command" ]]; then
    fail "$Usage"
  fi

  if ! [[ "$outfile" ]]; then
    fail "Error: output filename cannot be empty."
  fi
  if ! [[ -d "$(dirname "$outfile")" ]]; then
    fail "Error: Path invalid: $outfile"
  fi

  job_args=
  if [[ "$slurm" ]]; then
    if ! which srun >/dev/null 2>/dev/null; then
      fail "Error: srun command not found. Turn off slurm with -S."
    fi
    node=$(get_free_node)
    while ! [[ "$node" ]]; do
      if [[ "$debug" ]]; then
        echo "No slurm nodes are currently free. Waiting.." >&2
      fi
      sleep 60
      node=$(get_free_node)
    done
    if [[ "$debug" ]]; then
      echo "Launching command on $node" >&2
    fi
    if [[ "$job_name" ]]; then
      job_args="-J $job_name"
    fi
    if ! [[ -d ~/tmp ]]; then
      mkdir ~/tmp
    fi
    slurm_cmd="srun -w $node $slurm_args"
    monitor_prefix="ssh $node"
    monitor_selector="$command $command_args"
    time_file=$(tempfile -d ~/tmp --prefix time. --suffix .tsv)
    mem_file=$(tempfile -d ~/tmp --prefix mem. --suffix .tsv)
  else
    slurm_cmd=
    monitor_prefix=
    monitor_selector="-p $$"
    time_file=$(tempfile --prefix time. --suffix .tsv)
    mem_file=$(tempfile --prefix mem. --suffix .tsv)
  fi

  time_cmd=$(which time)<commit_msg>utils/performance/measure-cmd.sh: Remove leftover reference to $job_name.<commit_after>#!/usr/bin/env bash
if [ "x$BASH" = x ] || [ ! "$BASH_VERSINFO" ] || [ "$BASH_VERSINFO" -lt 4 ]; then
  echo "Error: Must use bash version 4+." >&2
  exit 1
fi
set -ue

Usage="Usage: \$ $(basename "$0") [options] command [command args]
Measure the resources used by a command and all of its children.
Prints 8 tab-delimited columns:
0. The -i argument, if given.
1. ps %CPU: The maximum percentage of CPU used at any one time.
2. ps %MEM: The maximum percentage of RAM used at any one time.
3. ps  VSZ: The maximum virtual memory size (in KB) used at any one time.
4. ps  RSS: The maximum resident set size (in KB) used at any one time.
5. time -f %e: Total number of elapsed wall clock seconds.
6. time -f %S: Total CPU-seconds used by the kernel on behalf of the command.
7. time -f %U: Total CPU-seconds of user-mode execution time.
8. time -f %M: Maximum resident set size in KB.
-i: A string that will be prepended to the output as the first column.
-o: Output file. The command's stdout will be piped here. Default: /dev/null
-l: Log file. The command's stderr will be piped here. Default: /dev/null
-s: Run under slurm. Prefixes the command with \"srun\".
-S: Arguments to give the slurm \"srun\" command.
-D: Turn on debug mode."

function main {

  # Get arguments.
  id=
  outfile=/dev/null
  logfile=/dev/null
  slurm=
  slurm_args=
  debug=
  while getopts "i:o:l:sS:Dh" opt; do
    case "$opt" in
      i) id="$OPTARG";;
      o) outfile="$OPTARG";;
      l) logfile="$OPTARG";;
      s) slurm=true;;
      S) slurm_args="$OPTARG";;
      D) debug='-D';;
      [h?]) fail "$Usage";;
    esac
  done
  command="${@:$OPTIND:1}"
  command_args="${@:$OPTIND+1}"

  if ! [[ "$command" ]]; then
    fail "$Usage"
  fi

  if ! [[ "$outfile" ]]; then
    fail "Error: output filename cannot be empty."
  fi
  if ! [[ -d "$(dirname "$outfile")" ]]; then
    fail "Error: Path invalid: $outfile"
  fi

  if [[ "$slurm" ]]; then
    if ! which srun >/dev/null 2>/dev/null; then
      fail "Error: srun command not found. Turn off slurm with -S."
    fi
    node=$(get_free_node)
    while ! [[ "$node" ]]; do
      if [[ "$debug" ]]; then
        echo "No slurm nodes are currently free. Waiting.." >&2
      fi
      sleep 60
      node=$(get_free_node)
    done
    if [[ "$debug" ]]; then
      echo "Launching command on $node" >&2
    fi
    if ! [[ -d ~/tmp ]]; then
      mkdir ~/tmp
    fi
    slurm_cmd="srun -w $node $slurm_args"
    monitor_prefix="ssh $node"
    monitor_selector="$command $command_args"
    time_file=$(tempfile -d ~/tmp --prefix time. --suffix .tsv)
    mem_file=$(tempfile -d ~/tmp --prefix mem. --suffix .tsv)
  else
    slurm_cmd=
    monitor_prefix=
    monitor_selector="-p $$"
    time_file=$(tempfile --prefix time. --suffix .tsv)
    mem_file=$(tempfile --prefix mem. --suffix .tsv)
  fi

  time_cmd=$(which time)
{'fraction_non_alphanumeric': 0.14825482887156896, 'fraction_numerical': 0.007455099966113182, 'mean_word_length': 2.7718849840255593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3607380', 'n_tokens_mistral': 2131, 'n_tokens_neox': 2070, 'n_words': 767}