<commit_before>#!/bin/sh
#

set -e

# Get the initial version.
sh version.sh

# check for sane defaults on some platforms
UNAME=`uname -s`
if [ -d /opt/local/share/aclocal ] && [ $UNAME = "Darwin" ] ; then
ACLOCALFLAGS=" -I /opt/local/share/aclocal"
fi

echo "libtoolize..."
if [ $UNAME = "Darwin" ]; then
glibtoolize --automake
else
libtoolize --automake
fi

echo "aclocal..."
ACLOCAL=`which aclocal-1.10 || which aclocal-1.9 || which aclocal19 || which aclocal-1.7 || which aclocal17 || which aclocal-1.5 || which aclocal15`
${ACLOCAL:-aclocal} $ACLOCALFLAGS || exit 1

echo "autoheader..."
AUTOHEADER=${AUTOHEADER:-autoheader}
$AUTOHEADER || exit 1

echo "automake..."
AUTOMAKE=`which automake-1.10 || which automake-1.9 || which automake-1.7`
$AUTOMAKE --foreign --add-missing || automake --foreign --add-missing || exit 1

echo "autoconf..."
AUTOCONF=${AUTOCONF:-autoconf}
$AUTOCONF || exit 1

# this is important so that this file is not rebuilt during builds
touch libmemcached-0.30/libmemcached/libmemcached_config.h.in

if [ -d libconflate ] && [ -d .git ] && ! [ -f libconflate/autogen.sh ]; then
  echo "The libconflate submodule seems to be absent."
  echo "Fetching submodules recursively."
  git submodule init && git submodule update &&
(cd libconflate && git submodule init && git submodule update) &&
(cd libconflate/libstrophe && git submodule init && git submodule update)
fi

# if this project is based on git, not just make-dist tarball,
# clean submodules recursively on autogen
for i in $(find . -name .git \
           | awk 'BEGIN{getline} \
           {print (substr($0, 0, index($0, ".git")-1 ))}');
do
  currdir=$(pwd)
  cd $i && git reset --hard > /dev/null 2>&1 && \
     git clean -d -f -x > /dev/null 2>&1
  cd $currdir
done

if (test -f libconflate/autogen.sh) && ! (test -f libconflate/configure); then
    echo "libconflate..."
    (cd libconflate && ./autogen.sh)
fi<commit_msg>Added additional checking to ensure all submodules have been added.<commit_after>#!/bin/sh
#

set -e

# Get the initial version.
sh version.sh

# check for sane defaults on some platforms
UNAME=`uname -s`
if [ -d /opt/local/share/aclocal ] && [ $UNAME = "Darwin" ] ; then
ACLOCALFLAGS=" -I /opt/local/share/aclocal"
fi

echo "libtoolize..."
if [ $UNAME = "Darwin" ]; then
glibtoolize --automake
else
libtoolize --automake
fi

echo "aclocal..."
ACLOCAL=`which aclocal-1.10 || which aclocal-1.9 || which aclocal19 || which aclocal-1.7 || which aclocal17 || which aclocal-1.5 || which aclocal15`
${ACLOCAL:-aclocal} $ACLOCALFLAGS || exit 1

echo "autoheader..."
AUTOHEADER=${AUTOHEADER:-autoheader}
$AUTOHEADER || exit 1

echo "automake..."
AUTOMAKE=`which automake-1.10 || which automake-1.9 || which automake-1.7`
$AUTOMAKE --foreign --add-missing || automake --foreign --add-missing || exit 1

echo "autoconf..."
AUTOCONF=${AUTOCONF:-autoconf}
$AUTOCONF || exit 1

# this is important so that this file is not rebuilt during builds
touch libmemcached-0.30/libmemcached/libmemcached_config.h.in

if [ -d libconflate ] && [ -d .git ] && \
   ( ! [ -f libconflate/libstrophe/.git ] || \
       [ -f libconflate/autogen.sh ] ); then
  echo "The libconflate submodule or it's submodules seem to be absent."
  echo "Fetching submodules recursively."
  git submodule init && git submodule update &&
(cd libconflate && git submodule init && git submodule update) &&
(cd libconflate/libstrophe && git submodule init && git submodule update)
fi

# if this project is based on git, not just make-dist tarball,
# clean submodules recursively on autogen
for i in $(find . -name .git \
           | awk 'BEGIN{getline} \
           {print (substr($0, 0, index($0, ".git")-1 ))}');
do
  currdir=$(pwd)
  cd $i && git reset --hard > /dev/null 2>&1 && \
     git clean -d -f -x > /dev/null 2>&1
  cd $currdir
done

if (test -f libconflate/autogen.sh) && ! (test -f libconflate/configure); then
    echo "libconflate..."
    (cd libconflate && ./autogen.sh)
fi
{'fraction_non_alphanumeric': 0.1506572295247725, 'fraction_numerical': 0.01870576339737108, 'mean_word_length': 3.411371237458194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1002529', 'n_tokens_mistral': 1431, 'n_tokens_neox': 1402, 'n_words': 427}