<commit_before>#!/bin/bash
# exit when any command fails
set -e
PYMONGO=$(dirname "$(cd "$(dirname "$0")"; pwd)")
SPECS=${MDB_SPECS:-~/Work/specifications}

help (){
  echo "Usage: resync_specs.sh [-bcsp] spec"
  echo "Required arguments:"
  echo " spec     determines which folder the spec tests will be copied from."
  echo "Optional flags:"
  echo " -b  is used to add a string to the blocklist for that next run. Can be used"
  echo "     any number of times on a single command to block multiple patterns."
  echo "     You can use any regex pattern (it is passed to 'grep -Ev')."
  echo " -c  is used to set a branch or commit that will be checked out in the"
  echo "     specifications repo before copying."
  echo " -s  is used to set a unique path to the specs repo for that specific"
  echo "     run."
  echo "Notes:"
  echo "You can export the environment variable MDB_SPECS to set the specs"
  echo " repo similar to -s, but this will persist between runs until you "
  echo "unset it."
}

# Parse flag args
BRANCH=''
BLOCKLIST='.*\.yml'
while getopts 'b:c:s:' flag; do
  case "${flag}" in
    b) BLOCKLIST+="|$OPTARG"
      ;;
    c) BRANCH="${OPTARG}"
      ;;
    s) SPECS="${OPTARG}"
      ;;
    *) help; exit 0
      ;;
  esac
done
shift $((OPTIND-1))

if [ -z $BRANCH ]
then
  git -C $SPECS checkout $BRANCH
fi

# Ensure the JSON files are up to date.
cd $SPECS/source
make
cd -
# cpjson unified-test-format/tests/invalid unified-test-format/invalid
# * param1: Path to spec tests dir in specifications repo
# * param2: Path to where the corresponding tests live in Python.
cpjson () {
    find "$PYMONGO"/test/$2 -type f -delete
    cd "$SPECS"/source/$1
    find . -name '*.json' | grep -Ev "${BLOCKLIST}" | cpio -pdm \
    $PYMONGO/test/$2
    printf "\nIgnored files for ${PWD}"
    printf "\n%s\n" "$(diff <(find . -name '*.json' | sort) \
    <(find . -name '*.json' | grep -Ev "${BLOCKLIST}" | sort))" | \
    sed -e '/^[0-9]/d' | sed -e 's|< ./||g'
}

for spec in "$@"
do
  case "$spec" in
    bson*corpus)
      cpjson bson-corpus/tests/ bson_corpus
      ;;
    max*staleness)
      cpjson max-staleness/tests/ max_staleness
      ;;
    connection*string)
      cpjson connection-string/tests/ connection_string/test
      ;;
    change*streams)
      cpjson change-streams/tests/ change_streams/
      ;;
    cmap|CMAP)
      cpjson connection-monitoring-and-pooling/tests cmap
      rm $PYMONGO/test/cmap/wait-queue-fairness.json  # PYTHON-1873<commit_msg>PYTHON-3136 [DevOps] Resync-specs.sh removes ignored files from working tree (#878)<commit_after>#!/bin/bash
# exit when any command fails
set -e
PYMONGO=$(dirname "$(cd "$(dirname "$0")"; pwd)")
SPECS=${MDB_SPECS:-~/Work/specifications}

help (){
  echo "Usage: resync_specs.sh [-bcsp] spec"
  echo "Required arguments:"
  echo " spec     determines which folder the spec tests will be copied from."
  echo "Optional flags:"
  echo " -b  is used to add a string to the blocklist for that next run. Can be used"
  echo "     any number of times on a single command to block multiple patterns."
  echo "     You can use any regex pattern (it is passed to 'grep -Ev')."
  echo " -c  is used to set a branch or commit that will be checked out in the"
  echo "     specifications repo before copying."
  echo " -s  is used to set a unique path to the specs repo for that specific"
  echo "     run."
  echo "Notes:"
  echo "You can export the environment variable MDB_SPECS to set the specs"
  echo " repo similar to -s, but this will persist between runs until you "
  echo "unset it."
}

# Parse flag args
BRANCH=''
BLOCKLIST='.*\.yml'
while getopts 'b:c:s:' flag; do
  case "${flag}" in
    b) BLOCKLIST+="|$OPTARG"
      ;;
    c) BRANCH="${OPTARG}"
      ;;
    s) SPECS="${OPTARG}"
      ;;
    *) help; exit 0
      ;;
  esac
done
shift $((OPTIND-1))

if [ -z $BRANCH ]
then
  git -C $SPECS checkout $BRANCH
fi

# Ensure the JSON files are up to date.
cd $SPECS/source
make
cd -
# cpjson unified-test-format/tests/invalid unified-test-format/invalid
# * param1: Path to spec tests dir in specifications repo
# * param2: Path to where the corresponding tests live in Python.
cpjson () {
    find "$PYMONGO"/test/$2 -type f -delete
    cd "$SPECS"/source/$1
    find . -name '*.json' | grep -Ev "${BLOCKLIST}" | cpio -pdm \
    $PYMONGO/test/$2
    printf "\nIgnored files for ${PWD}\n"
    IGNORED_FILES="$(printf "\n%s\n" "$(diff <(find . -name '*.json' | sort) \
    <(find . -name '*.json' | grep -Ev "${BLOCKLIST}" | sort))" | \
    sed -e '/^[0-9]/d' | sed -e 's|< ./||g' )"
    printf "%s\n" $IGNORED_FILES
    cd "$PYMONGO"/test/$2
    printf "%s\n" $IGNORED_FILES | xargs git checkout master

}

for spec in "$@"
do
  case "$spec" in
    bson*corpus)
      cpjson bson-corpus/tests/ bson_corpus
      ;;
    max*staleness)
      cpjson max-staleness/tests/ max_staleness
      ;;
    connection*string)
      cpjson connection-string/tests/ connection_string/test
      ;;
    change*streams)
      cpjson change-streams/tests/ change_streams/
      ;;
    cmap|CMAP)
      cpjson connection-monitoring-and-pooling/tests cmap
      rm $PYMONGO/test/cmap/wait-queue-fairness.json  # PYTHON-1873
{'fraction_non_alphanumeric': 0.1359522434045831, 'fraction_numerical': 0.006932409012131715, 'mean_word_length': 2.816311535635562, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6406501', 'n_tokens_mistral': 1858, 'n_tokens_neox': 1767, 'n_words': 610}