<commit_before>#!/bin/bash
#
# SyncSiteWithRepo - synchronize the examples site with the
#                    examples repo
#
if [ $# -lt 1 ]
  then
  echo "Usage: SyncSiteWithRepo REPO_URL"
  exit 1
fi
REPO=$1
# Make sure the repo site is up
echo "Synchronizing the VTKExamples site with the repository."
HOST=www.github.com
echo "0) Can we access the repo?"
ping -c 1 $HOST &> /dev/null
if test "${?}" != 0
  then
  echo "VTKExamples: $HOST is not accessible. Try again later" | mail -s "SyncSiteWithRepo failed" <EMAIL>
  exit 1
fi

echo "1) Pull updates from master repositories"
git pull
if ( test -d Tarballs ); then
  (cd Tarballs; git checkout .)
  (cd Tarballs; git pull origin master)
fi

echo "2) Scrape the repo"
rm -rf docs/*
rm -rf site/*
src/Admin/ScrapeRepo  src docs ${REPO}

echo "3) Check for a successful scrape"
pushd docs
count=$((`find . -name \*.md | wc -l`))
popd
expected=800
if test $count -lt $expected; then
   echo VTKExamples/Admin/ScrapeRepo failed
   echo VTKExamples: Expected at least $expected md files but only found $count md | mail -s "SyncSiteWithRepo failed" <EMAIL>
   git checkout .
   exit 1
fi

echo "4) Update the html pages"
(cd site; rm -rf *)
mkdocs build

echo "5) Copy sitemap.xml"
cp src/Admin/sitemap.xml site/sitemap.xml

echo "5.1) Process modified files"
git commit -m "SYNC: Files modified in the repo." `git status | grep modified: | cut -d":" -f2,2`

echo "5.2) Process new files"
find . "(" -name \*.html ")" -exec git add {} \;
git commit -m "SYNC: Files added to the repo."

echo "5.3) Process deleted files"
git rm `git status | grep deleted: | cut -d":" -f2,2`
git commit -m "SYNC: Files deleted (or moved) from the repo."

echo "6) Update tarballs and push to tarball repo"
if ( test -d src/Tarballs ); then
(cd src/Tarballs; rm *.tar)
(cd src/Tarballs; git add *tar)
(cd src/Tarballs; git commit -m "SYNC: Tarballs modified")
(cd src/Tarballs; git push)
fi

echo "7) Push the changes"
git push
<commit_msg>BUG: Mishandling of tarballs<commit_after>#!/bin/bash
#
# SyncSiteWithRepo - synchronize the examples site with the
#                    examples repo
#
if [ $# -lt 1 ]
  then
  echo "Usage: SyncSiteWithRepo REPO_URL"
  exit 1
fi
REPO=$1
# Make sure the repo site is up
echo "Synchronizing the VTKExamples site with the repository."
HOST=www.github.com
echo "0) Can we access the repo?"
ping -c 1 $HOST &> /dev/null
if test "${?}" != 0
  then
  echo "VTKExamples: $HOST is not accessible. Try again later" | mail -s "SyncSiteWithRepo failed" <EMAIL>
  exit 1
fi

echo "1) Pull updates from master repositories"
git pull
if ( test -d src/Tarballs ); then
  (cd src/Tarballs; git checkout .)
  (cd src/Tarballs; git pull origin master)
  (cd src/Tarballs; rm *.tar)
fi

echo "2) Scrape the repo"
rm -rf docs/*
rm -rf site/*
src/Admin/ScrapeRepo  src docs ${REPO}

echo "3) Check for a successful scrape"
pushd docs
count=$((`find . -name \*.md | wc -l`))
popd
expected=800
if test $count -lt $expected; then
   echo VTKExamples/Admin/ScrapeRepo failed
   echo VTKExamples: Expected at least $expected md files but only found $count md | mail -s "SyncSiteWithRepo failed" <EMAIL>
   git checkout .
   exit 1
fi

echo "4) Update the html pages"
(cd site; rm -rf *)
mkdocs build

echo "5) Copy sitemap.xml"
cp src/Admin/sitemap.xml site/sitemap.xml

echo "5.1) Process modified files"
git commit -m "SYNC: Files modified in the repo." `git status | grep modified: | cut -d":" -f2,2`

echo "5.2) Process new files"
find . "(" -name \*.html ")" -exec git add {} \;
git commit -m "SYNC: Files added to the repo."

echo "5.3) Process deleted files"
git rm `git status | grep deleted: | cut -d":" -f2,2`
git commit -m "SYNC: Files deleted (or moved) from the repo."

echo "6) Update tarballs and push to tarball repo"
if ( test -d src/Tarballs ); then
(cd src/Tarballs; git add *tar)
(cd src/Tarballs; git commit -m "SYNC: Tarballs modified")
(cd src/Tarballs; git push)
fi

echo "7) Push the changes"
git push

{'fraction_non_alphanumeric': 0.12131890259249938, 'fraction_numerical': 0.01409514220991694, 'mean_word_length': 3.4601571268237934, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479781', 'n_tokens_mistral': 1462, 'n_tokens_neox': 1400, 'n_words': 577}