<commit_before>#!/bin/sh

# tests that admin urls are mapped to public
# and that urls which appear first trump older urls

test_document_mappings='/tmp/munge_test_document_mappings.csv'
fetched_data='/tmp/munge_test_fetched_data.csv'
output='/tmp/munge_test.out'

run_munge () {
  cat $fetched_data |
  ./munge/munge.rb $test_document_mappings 2>/dev/null |
  ./tools/fold-mappings.rb |
  ./tools/choose-status.rb |
  ./munge/strip-empty-quotes-and-whitespace.rb |
  ./munge/reverse-csv.rb |
  ./tools/tidy_mappings.pl --trump > $output 2> /dev/null
}

cat > $test_document_mappings <<!
Old Url,New Url,Status,Whole Tag,Slug,Admin Url,State
"",https://www.gov.uk/government/policies/remapped-public-url,"",Open,remapped-public-url,https://whitehall-admin.production.alphagov.co.uk/government/admin/policy_advisory_groups/88/edit,archived
!

cat > $fetched_data <<!
old url,new url,status,source,row_number
http://www.decc.gov.uk,https://whitehall-admin.production.alphagov.co.uk/government/admin/policy_advisory_groups/88/edit,,passed in string,1
http://www.decc.gov.uk,https://gov.uk/this-should-be-trumped-and-not-appear,,passed in string,2
http://www.decc.gov.uk/foo,https://gov.uk/this-is-foo,,passed in string,3
http://www.decc.gov.uk/foo?,https://gov.uk/this-should-not-appear-as-foo,,passed in string,3
http://www.decc.gov.uk/foo#,https://gov.uk/this-should-not-appear-as-foo,,passed in string,3
!

run_munge

diff $output - <<!
Old Url,New Url,Status
http://www.decc.gov.uk/foo,https://gov.uk/this-is-foo,301
!

[ $? -ne 0 ] && { echo "$0: FAIL" ; exit 1; }

# test: ensure that different cases are treated correctly with trumping

cat > $fetched_data <<!
old url,new url,status,source,row_number
http://www.decc.gov.uk/foo,https://gov.uk/this-is-foo,,passed in string,3
http://www.decc.gov.uk/FOO,https://gov.uk/this-should-not-appear-as-foo-as-capitalised,,passed in string,3
!<commit_msg>Ensure intent of test remains the same<commit_after>#!/bin/sh

# tests that admin urls are mapped to public
# and that urls which appear first trump older urls

test_document_mappings='/tmp/munge_test_document_mappings.csv'
fetched_data='/tmp/munge_test_fetched_data.csv'
output='/tmp/munge_test.out'

run_munge () {
  cat $fetched_data |
  ./munge/munge.rb $test_document_mappings 2>/dev/null |
  ./tools/fold-mappings.rb |
  ./tools/choose-status.rb |
  ./munge/strip-empty-quotes-and-whitespace.rb |
  ./munge/reverse-csv.rb |
  ./tools/tidy_mappings.pl --trump > $output 2> /dev/null
}

cat > $test_document_mappings <<!
Old Url,New Url,Status,Whole Tag,Slug,Admin Url,State
"",https://www.gov.uk/government/policies/remapped-public-url,"",Open,remapped-public-url,https://whitehall-admin.production.alphagov.co.uk/government/admin/policy_advisory_groups/88/edit,archived
!

cat > $fetched_data <<!
old url,new url,status,source,row_number
http://www.decc.gov.uk/bar,https://whitehall-admin.production.alphagov.co.uk/government/admin/policy_advisory_groups/88/edit,,passed in string,1
http://www.decc.gov.uk/bar,https://gov.uk/this-should-be-trumped-and-not-appear,,passed in string,2
http://www.decc.gov.uk/foo,https://gov.uk/this-is-foo,,passed in string,3
http://www.decc.gov.uk/foo?,https://gov.uk/this-should-not-appear-as-foo,,passed in string,3
http://www.decc.gov.uk/foo#,https://gov.uk/this-should-not-appear-as-foo,,passed in string,3
!

run_munge

diff $output - <<!
Old Url,New Url,Status
http://www.decc.gov.uk/bar,https://www.gov.uk/government/policies/remapped-public-url,301
http://www.decc.gov.uk/foo,https://gov.uk/this-is-foo,301
!

[ $? -ne 0 ] && { echo "$0: FAIL" ; exit 1; }

# test: ensure that different cases are treated correctly with trumping

cat > $fetched_data <<!
old url,new url,status,source,row_number
http://www.decc.gov.uk/foo,https://gov.uk/this-is-foo,,passed in string,3
http://www.decc.gov.uk/FOO,https://gov.uk/this-should-not-appear-as-foo-as-capitalised,,passed in string,3
!
{'fraction_non_alphanumeric': 0.1761407086413459, 'fraction_numerical': 0.010451185317359163, 'mean_word_length': 4.534555712270804, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 21, 'lorem ipsum': 0, 'www.': 20, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1995963', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1527, 'n_words': 205}