<commit_before>echo "${configuration}" | shyaml get-values-0 websites.apache |
while IFS='' read -r -d '' key; do

    domain=$(echo "$key" | grep -w "domain" | cut -d ":" -f 2 | tr -d " ")
    software=$(echo "$key" | grep -w "software" | cut -d ":" -f 2 | tr -d " ")
    software_workflow=$(echo "$key" | grep -w "software_workflow" | cut -d ":" -f 2 | tr -d " ")
    webroot=$(echo "$key" | grep -w "webroot" | cut -d ":" -f 2 | tr -d " ")

    echo -e "\nNOTICE: $domain"

    if [ "${software}" = "drupal6" ]; then
        echo -e "\t * rysncing ${software} ~/sites/default/files"
        if ([ "${software_workflow}" = "downstream" ] && [ "$1" != "production" ]); then
            sudo rsync  --archive --compress --copy-links --delete -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.production.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files /var/www/repositories/apache/${domain}/${webroot}sites/default/files 2>&1 | sed "s/^/\t\t/"
        elif ([ "${software_workflow}" = "upstream" ] && [ "$1" != "test" ]); then
            sudo rsync  --archive --compress --copy-links --delete -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.test.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files /var/www/repositories/apache/${domain}/${webroot}sites/default/files 2>&1 | sed "s/^/\t\t/"
        fi
    elif [ "${software}" = "drupal7" ]; then
        echo -e "\t * rysncing ${software} ~/sites/default/files"
        if ([ "${software_workflow}" = "downstream" ] && [ "$1" != "production" ]); then
            sudo rsync  --archive --compress --copy-links --delete -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.production.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files /var/www/repositories/apache/${domain}/${webroot}sites/default/files 2>&1 | sed "s/^/\t\t/"
        elif ([ "${software_workflow}" = "upstream" ] && [ "$1" != "test" ]); then
            sudo rsync  --archive --compress --copy-links --delete -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.test.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files /var/www/repositories/apache/${domain}/${webroot}sites/default/files 2>&1 | sed "s/^/\t\t/"
        fi
    elif [ "${software}" = "wordpress" ]; then
        echo -e "\t * rysncing ${software} ~/wp-content/uploads"
        if ([ "${software_workflow}" = "downstream" ] && [ "$1" != "production" ]); then
            sudo rsync  --archive --compress --copy-links --delete -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.production.servers.redhat.ip):/var/www/repositories/apache/${domain}/${webroot}wp-content/uploads /var/www/repositories/apache/${domain}/${webroot}wp-content/uploads 2>&1 | sed "s/^/\t\t/"
        elif ([ "${software_workflow}" = "upstream" ] && [ "$1" != "test" ]); then
            sudo rsync  --archive --compress --copy-links --delete -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.test.servers.redhat.ip):/var/www/repositories/apache/${domain}/${webroot}wp-content/uploads /var/www/repositories/apache/${domain}/${webroot}wp-content/uploads 2>&1 | sed "s/^/\t\t/"
        fi
    else
        echo -e "\t * no rsync needed, skipping"
    fi

done<commit_msg>--archive carries too much historical, including --group and --owner, lets be specific<commit_after>echo "${configuration}" | shyaml get-values-0 websites.apache |
while IFS='' read -r -d '' key; do

    domain=$(echo "$key" | grep -w "domain" | cut -d ":" -f 2 | tr -d " ")
    software=$(echo "$key" | grep -w "software" | cut -d ":" -f 2 | tr -d " ")
    software_workflow=$(echo "$key" | grep -w "software_workflow" | cut -d ":" -f 2 | tr -d " ")
    webroot=$(echo "$key" | grep -w "webroot" | cut -d ":" -f 2 | tr -d " ")

    echo -e "\nNOTICE: $domain"

    if [ "${software}" = "drupal6" ]; then
        echo -e "\t * rysncing ${software} ~/sites/default/files"
        if ([ "${software_workflow}" = "downstream" ] && [ "$1" != "production" ]); then
            sudo rsync --compress --delete --recursive -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.production.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files/ /var/www/repositories/apache/${domain}/${webroot}sites/default/files/ 2>&1 | sed "s/^/\t\t/"
        elif ([ "${software_workflow}" = "upstream" ] && [ "$1" != "test" ]); then
            sudo rsync --compress --delete --recursive -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.test.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files/ /var/www/repositories/apache/${domain}/${webroot}sites/default/files/ 2>&1 | sed "s/^/\t\t/"
        fi
    elif [ "${software}" = "drupal7" ]; then
        echo -e "\t * rysncing ${software} ~/sites/default/files"
        if ([ "${software_workflow}" = "downstream" ] && [ "$1" != "production" ]); then
            sudo rsync --compress --delete --recursive -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.production.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files/ /var/www/repositories/apache/${domain}/${webroot}sites/default/files/ 2>&1 | sed "s/^/\t\t/"
        elif ([ "${software_workflow}" = "upstream" ] && [ "$1" != "test" ]); then
            sudo rsync --compress --delete --recursive -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.test.servers.redhat.ip):/var/www/repositories/apache/${domain}/sites/default/files/ /var/www/repositories/apache/${domain}/${webroot}sites/default/files/ 2>&1 | sed "s/^/\t\t/"
        fi
    elif [ "${software}" = "wordpress" ]; then
        echo -e "\t * rysncing ${software} ~/wp-content/uploads"
        if ([ "${software_workflow}" = "downstream" ] && [ "$1" != "production" ]); then
            sudo rsync --compress --delete --recursive -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.production.servers.redhat.ip):/var/www/repositories/apache/${domain}/${webroot}wp-content/uploads/ /var/www/repositories/apache/${domain}/${webroot}wp-content/uploads/ 2>&1 | sed "s/^/\t\t/"
        elif ([ "${software_workflow}" = "upstream" ] && [ "$1" != "test" ]); then
            sudo rsync --compress --delete --recursive -e "ssh -oStrictHostKeyChecking=no -i /catapult/secrets/id_rsa" root@$(echo "${configuration}" | shyaml get-value environments.test.servers.redhat.ip):/var/www/repositories/apache/${domain}/${webroot}wp-content/uploads/ /var/www/repositories/apache/${domain}/${webroot}wp-content/uploads/ 2>&1 | sed "s/^/\t\t/"
        fi
    else
        echo -e "\t * no rsync needed, skipping"
    fi

done
{'fraction_non_alphanumeric': 0.20489917444850453, 'fraction_numerical': 0.006766815536608472, 'mean_word_length': 3.2766203703703702, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 24, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1992720', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2494, 'n_words': 522}