<commit_before>#!/usr/bin/env bats

# Tests written by <NAME>, August 2016

load '../test/test_helper/bats-support/load'
load '../test/test_helper/bats-assert/load'
load '../test/test_helper/bats-file/load'

little=little_dir
num_little_remaining_files=16

big=big_dir
num_big_remaining_files=792

setup() {
  # Create a temporary scratch directory for the shell script to work in.
  BATS_TMPDIR=$(temp_make --prefix 'compiling-')
  # The next two lines simplify the display of the potentially long, 
  # complex paths generated by `temp_make` so the output of `bats` is
  # hopefully more readable. 
  # shellcheck disable=SC2034
  BATSLIB_FILE_PATH_REM="#${BATS_TMPDIR}"
  # shellcheck disable=SC2034
  BATSLIB_FILE_PATH_ADD='<temp>'

  # Uncomment out the next line if you want to see where the temp files
  # are being created.

  # echo "$BATS_TMPDIR"

  # Uncommend out the next line if you want bats to preserve (i.e., not delete)
  # the temp files generated for failing tests. This might be useful in trying
  # to figure out what happens when a test fails.

  # shellcheck disable=SC2034
  BATSLIB_TEMP_PRESERVE_ON_FAILURE=1
}

# Remove the temporary scratch directory to clean up after ourselves.
teardown() {
  temp_del "$BATS_TMPDIR"
}

# If this test fails, your script file doesn't exist, or there's
# a typo in the name, or it's in the wrong directory, etc.
@test "big_clean.sh exists" {
  assert_file_exist big_clean.sh
}

# If this test fails, your script isn't executable.
@test "big_clean.sh is executable" {
  assert_file_executable big_clean.sh
}

# If this test fails, your script either didn't run at all, or it
# generated some sort of error when it ran.
@test "big_clean.sh runs successfully" {
  run ./big_clean.sh $little.tgz "$BATS_TMPDIR"
  assert_success
}

# If this test fails, you either didn't extract the contents of the
# `tar` archive, or you extracted them into the wrong directory. If you're
# having trouble debugging this, you might find it useful to call your
# script directly from the command line and see where it extracted the files.
@test "big_clean.sh extracts the 'tar' archive contents" {
  run ./big_clean.sh $little.tgz "$BATS_TMPDIR"
  assert_dir_exist "$BATS_TMPDIR/$little"
  # This just checks that a few of the files are there.
  assert_file_exist "$BATS_TMPDIR/$little/file_6"
  assert_file_exist "$BATS_TMPDIR/$little/file_12"
  assert_file_exist "$BATS_TMPDIR/$little/file_18"
}

# If this test fails, you either moved or renamed the compressed `tar` archive.
# One common way this can happen is if you used `gunzip` to uncompressed the
# archive, and then used `tar xf` to extract the contents in a separate step.
# That would leave the archive as `NthPrime.tar` instead of `NthPrime.tgz`.
@test "big_clean.sh doesn't remove or rename the compressed 'tar' archive" {
  run ./big_clean.sh "$little.tgz" "$BATS_TMPDIR"
  assert_file_exist "$little.tgz"
}

@test "big_clean.sh creates new 'cleaned_little_dir.tgz' archive" {
  run ./big_clean.sh "$little.tgz" "$BATS_TMPDIR"
  assert_file_exist "cleaned_$little.tgz"
}

@test "The new archive has the right number of files in it" {
  ./big_clean.sh "$little.tgz" "$BATS_TMPDIR"
  run bash -c "tar -ztf cleaned_$little.tgz | grep '/f' | wc -l"
  assert_output --regexp "\s*$num_little_remaining_files\s*"
  # [ "$output" -eq $num_little_remaining_files ]
}

# We should be able to use `refute_line` to also say that some
# specific files should not be in the output.
@test "The new archive has (some) of the right files in it" {
  ./big_clean.sh $little.tgz "$BATS_TMPDIR"
  # run bash -c "tar -ztf cleaned_$little.tgz | grep '/f' | sort"
  run tar -ztf cleaned_$little.tgz
  # assert_line --index 0 "little_dir/file_1"
  # assert_line --index 1 "little_dir/file_10"
  # assert_line --index 8 "little_dir/file_19"
  assert_line --regexp "^little_dir/file_1$"
  assert_line --regexp "^little_dir/file_10$"
  assert_line --regexp "^little_dir/file_19$"
#  [ "${lines[0]}" == "little_dir/file_1" ]
#  [ "${lines[1]}" == "little_dir/file_10" ]
#  [ "${lines[8]}" == "little_dir/file_19" ]
}

@test "big_clean.sh returns the right number of files on the big archive" {
  run ./big_clean.sh $big.tgz "$BATS_TMPDIR"
  run bash -c "tar -ztf cleaned_$big.tgz | grep '/f' | wc -l"
  assert_output "$num_big_remaining_files"
  # [ "$output" -eq $num_big_remaining_files ]
}
<commit_msg>Expand the grep test for matching archive files<commit_after>#!/usr/bin/env bats

# Tests written by <NAME>, August 2016

load '../test/test_helper/bats-support/load'
load '../test/test_helper/bats-assert/load'
load '../test/test_helper/bats-file/load'

little=little_dir
num_little_remaining_files=16

big=big_dir
num_big_remaining_files=792

setup() {
  # Create a temporary scratch directory for the shell script to work in.
  BATS_TMPDIR=$(temp_make --prefix 'compiling-')
  # The next two lines simplify the display of the potentially long, 
  # complex paths generated by `temp_make` so the output of `bats` is
  # hopefully more readable. 
  # shellcheck disable=SC2034
  BATSLIB_FILE_PATH_REM="#${BATS_TMPDIR}"
  # shellcheck disable=SC2034
  BATSLIB_FILE_PATH_ADD='<temp>'

  # Uncomment out the next line if you want to see where the temp files
  # are being created.

  # echo "$BATS_TMPDIR"

  # Uncommend out the next line if you want bats to preserve (i.e., not delete)
  # the temp files generated for failing tests. This might be useful in trying
  # to figure out what happens when a test fails.

  # shellcheck disable=SC2034
  BATSLIB_TEMP_PRESERVE_ON_FAILURE=1
}

# Remove the temporary scratch directory to clean up after ourselves.
teardown() {
  temp_del "$BATS_TMPDIR"
}

# If this test fails, your script file doesn't exist, or there's
# a typo in the name, or it's in the wrong directory, etc.
@test "big_clean.sh exists" {
  assert_file_exist big_clean.sh
}

# If this test fails, your script isn't executable.
@test "big_clean.sh is executable" {
  assert_file_executable big_clean.sh
}

# If this test fails, your script either didn't run at all, or it
# generated some sort of error when it ran.
@test "big_clean.sh runs successfully" {
  run ./big_clean.sh $little.tgz "$BATS_TMPDIR"
  assert_success
}

# If this test fails, you either didn't extract the contents of the
# `tar` archive, or you extracted them into the wrong directory. If you're
# having trouble debugging this, you might find it useful to call your
# script directly from the command line and see where it extracted the files.
@test "big_clean.sh extracts the 'tar' archive contents" {
  run ./big_clean.sh $little.tgz "$BATS_TMPDIR"
  assert_dir_exist "$BATS_TMPDIR/$little"
  # This just checks that a few of the files are there.
  assert_file_exist "$BATS_TMPDIR/$little/file_6"
  assert_file_exist "$BATS_TMPDIR/$little/file_12"
  assert_file_exist "$BATS_TMPDIR/$little/file_18"
}

# If this test fails, you either moved or renamed the compressed `tar` archive.
# One common way this can happen is if you used `gunzip` to uncompressed the
# archive, and then used `tar xf` to extract the contents in a separate step.
# That would leave the archive as `NthPrime.tar` instead of `NthPrime.tgz`.
@test "big_clean.sh doesn't remove or rename the compressed 'tar' archive" {
  run ./big_clean.sh "$little.tgz" "$BATS_TMPDIR"
  assert_file_exist "$little.tgz"
}

@test "big_clean.sh creates new 'cleaned_little_dir.tgz' archive" {
  run ./big_clean.sh "$little.tgz" "$BATS_TMPDIR"
  assert_file_exist "cleaned_$little.tgz"
}

@test "The new archive has the right number of files in it" {
  ./big_clean.sh "$little.tgz" "$BATS_TMPDIR"
  run bash -c "tar -ztf cleaned_$little.tgz | egrep '^little_dir/file_\d+$' | wc -l"
  assert_output --regexp "\s*$num_little_remaining_files\s*"
  # [ "$output" -eq $num_little_remaining_files ]
}

# We should be able to use `refute_line` to also say that some
# specific files should not be in the output.
@test "The new archive has (some) of the right files in it" {
  ./big_clean.sh $little.tgz "$BATS_TMPDIR"
  # run bash -c "tar -ztf cleaned_$little.tgz | grep '/f' | sort"
  run tar -ztf cleaned_$little.tgz
  # assert_line --index 0 "little_dir/file_1"
  # assert_line --index 1 "little_dir/file_10"
  # assert_line --index 8 "little_dir/file_19"
  assert_line --regexp "^little_dir/file_1$"
  assert_line --regexp "^little_dir/file_10$"
  assert_line --regexp "^little_dir/file_19$"
#  [ "${lines[0]}" == "little_dir/file_1" ]
#  [ "${lines[1]}" == "little_dir/file_10" ]
#  [ "${lines[8]}" == "little_dir/file_19" ]
}

@test "big_clean.sh returns the right number of files on the big archive" {
  run ./big_clean.sh $big.tgz "$BATS_TMPDIR"
  run bash -c "tar -ztf cleaned_$big.tgz | egrep '^big_dir/file_\d+$' | wc -l"
  assert_output "$num_big_remaining_files"
  # [ "$output" -eq $num_big_remaining_files ]
}

{'fraction_non_alphanumeric': 0.10976575761004866, 'fraction_numerical': 0.010863415186149146, 'mean_word_length': 4.211084905660377, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2320990', 'n_tokens_mistral': 3237, 'n_tokens_neox': 3019, 'n_words': 1013}