<commit_before>#!/bin/bash -e

WWWDIR="`dirname \"$0\"`"
ensure ()
{
  LOCALFILE="$WWWDIR/$2/`basename \"$1\"`"
  test -f "$LOCALFILE" || wget -O "$LOCALFILE" "$1"
}

# Download files necessary to preview the web pages locally.
if [ ! -r "$WWWDIR/tigris-branding/.download-complete" ]; then
  BRANDING_URL="http://subversion.tigris.org/branding"
  mkdir -p "$WWWDIR/tigris-branding/"{css,scripts,images}
  for i in tigris inst print; do
    ensure "$BRANDING_URL/css/$i.css" "tigris-branding/css"
  done
  ensure "$BRANDING_URL/scripts/tigris.js" "tigris-branding/scripts"
  for f in `sed -n -e 's,.*url(\.\./images/\([^)]*\).*,\1,;tp' \
    -etp -ed -e:p -ep $WWWDIR/tigris-branding/css/*.css`; do
    ensure "$BRANDING_URL/images/$f" "tigris-branding/images"
  done
  touch "$WWWDIR/tigris-branding/.download-complete"
fi

# Check we have DTDs available
LOCAL_CATALOG="$WWWDIR/xhtml1.catalog"
if [ ! -r "$LOCAL_CATALOG" ]; then
  RESULT=`echo 'resolve "-//W3C//DTD XHTML 1.0 Strict//EN" ' \
  '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"' | xmlcatalog --shell`
  case $RESULT in
    */xhtml1-strict.dtd*) ;;
    *)
    ensure "http://www.w3.org/TR/xhtml1/xhtml1.tgz" "."
    rm -rf "$WWWDIR/xhtml1-20020801" "$LOCAL_CATALOG"
    tar -zxvf "$WWWDIR/xhtml1.tgz"
    xmlcatalog --noout --create "$LOCAL_CATALOG"
    xmlcatalog --noout --add rewriteSystem "http://www.w3.org/TR/xhtml1/" \
    "`cd \"$WWWDIR\" && pwd`/xhtml1-20020801/" "$LOCAL_CATALOG"
    ;;
  esac
fi<commit_msg>* www/validate.sh: Avoid erroring on a couple of images which are referenced     in tigris.css, but give 404s.<commit_after>#!/bin/bash -e

WWWDIR="`dirname \"$0\"`"
ensure ()
{
  LOCALFILE="$WWWDIR/$2/`basename \"$1\"`"
  test -f "$LOCALFILE" || wget -O "$LOCALFILE" "$1"
}

# Download files necessary to preview the web pages locally.
if [ ! -r "$WWWDIR/tigris-branding/.download-complete" ]; then
  BRANDING_URL="http://subversion.tigris.org/branding"
  mkdir -p "$WWWDIR/tigris-branding/"{css,scripts,images}
  for i in tigris inst print; do
    ensure "$BRANDING_URL/css/$i.css" "tigris-branding/css"
  done
  ensure "$BRANDING_URL/scripts/tigris.js" "tigris-branding/scripts"
  for f in `sed -n -e 's,.*url(\.\./images/\([^)]*\).*,\1,;tp' \
    -etp -ed -e:p -ep $WWWDIR/tigris-branding/css/*.css`; do
    case $f in
      collapsed_big.gif|expanded_big.gif) ;; # 404!
      *) ensure "$BRANDING_URL/images/$f" "tigris-branding/images" ;;
    esac
  done
  touch "$WWWDIR/tigris-branding/.download-complete"
fi

# Check we have DTDs available
LOCAL_CATALOG="$WWWDIR/xhtml1.catalog"
if [ ! -r "$LOCAL_CATALOG" ]; then
  RESULT=`echo 'resolve "-//W3C//DTD XHTML 1.0 Strict//EN" ' \
  '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"' | xmlcatalog --shell`
  case $RESULT in
    */xhtml1-strict.dtd*) ;;
    *)
    ensure "http://www.w3.org/TR/xhtml1/xhtml1.tgz" "."
    rm -rf "$WWWDIR/xhtml1-20020801" "$LOCAL_CATALOG"
    tar -zxvf "$WWWDIR/xhtml1.tgz"
    xmlcatalog --noout --create "$LOCAL_CATALOG"
    xmlcatalog --noout --add rewriteSystem "http://www.w3.org/TR/xhtml1/" \
    "`cd \"$WWWDIR\" && pwd`/xhtml1-20020801/" "$LOCAL_CATALOG"
    ;;
  esac
fi
{'fraction_non_alphanumeric': 0.1958795562599049, 'fraction_numerical': 0.025356576862123614, 'mean_word_length': 3.2591093117408905, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 7, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '756477', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1259, 'n_words': 245}