<commit_before>#! /bin/sh
# $Id$

XML_VERSION='4.4'
XML_FILE="docbook-xml-${XML_VERSION}"
XML_URL="http://www.docbook.org/xml/${XML_VERSION}/${XML_FILE}.zip"

XSL_VERSION='1.71.0'
XSL_FILE="docbook-xsl-${XSL_VERSION}"
XSL_URL="http://prdownloads.sourceforge.net/docbook/${XSL_FILE}.tar.bz2?download"

#-----------------------------------------------------------------------------
# do no edit below here
#-----------------------------------------------------------------------------

PATH="${PATH}:${MY_DIR}"
PWD=${PWD:-`pwd`}

MY_BASE=`basename $0`
MY_DIR=`dirname "$0"`

# load shlib
. "${MY_DIR}/../lib/sh/shlib"

BASE_DIR=`shlib_relToAbsPath "${MY_DIR}/.."`
DL_DIR="${BASE_DIR}/tmp"
DOCBOOK_DIR="${BASE_DIR}/share/docbook"

CURL_OPTS='-C - -Os'
WGET_OPTS='-cq'

METHOD_NONE=0
METHOD_WGET=1
METHOD_CURL=2

get_url()
{
  url=$1
  case ${method} in
    ${METHOD_CURL}) ${curl} ${CURL_OPTS} "${url}" ;;
    ${METHOD_WGET}) ${wget} ${WGET_OPTS} "${url}" ;;
  esac
}

# determine method
method=${METHOD_NONE}
wget=`which wget`
[ $? -eq 0 ] && method=${METHOD_WGET}
curl=`which curl`
[ $? -eq 0 -a ${method} -eq ${METHOD_NONE} ] && method=${METHOD_CURL}
if [ ${method} -eq ${METHOD_NONE} ]; then
  echo "unable to locate wget or curl. cannot continue"
  exit 1
fi

# create download dir
mkdir -p "${DL_DIR}"

# get the docbook xml files
echo 'Docbook XML'
echo '  downloading'
cd ${DL_DIR}
get_url "${XML_URL}"
echo '  extracting'
xml_dir="${DOCBOOK_DIR}/docbook-xml/${XML_VERSION}"
rm -fr "${xml_dir}"
mkdir -p "${xml_dir}"
cd "${xml_dir}"
unzip -oq "${DL_DIR}/${XML_FILE}.zip"
cd ..
rm current
ln -s "${XML_VERSION}" current

# get the docbook xslt files
echo 'Docbook XSLT'
echo '  downloading'
cd ${DL_DIR}
get_url "${XSL_URL}"
echo '  extracting'
xsl_dir="${DOCBOOK_DIR}/docbook-xsl"
mkdir -p "${xsl_dir}"
cd "${xsl_dir}"
rm -fr ${XSL_VERSION}
bzip2 -dc ${DL_DIR}/${XSL_FILE}.tar.bz2 |tar xf -
mv ${XSL_FILE} ${XSL_VERSION}
rm current
ln -s "${XSL_VERSION}" current
<commit_msg>fixed issues with the docbook preparation<commit_after>#! /bin/sh
# $Id$

XML_VERSION='4.4'
XML_FILE="docbook-xml-${XML_VERSION}"
XML_URL="http://www.docbook.org/xml/${XML_VERSION}/${XML_FILE}.zip"

XSL_VERSION='1.71.0'
XSL_FILE="docbook-xsl-${XSL_VERSION}"
XSL_URL="http://prdownloads.sourceforge.net/docbook/${XSL_FILE}.tar.bz2?download"

#-----------------------------------------------------------------------------
# do no edit below here
#-----------------------------------------------------------------------------

PATH="${PATH}:${MY_DIR}"
PWD=${PWD:-`pwd`}

MY_BASE=`basename $0`
MY_DIR=`dirname "$0"`

# load shlib
. "${MY_DIR}/../lib/sh/shlib"

BASE_DIR=`shlib_relToAbsPath "${MY_DIR}/.."`
DL_DIR="${BASE_DIR}/tmp"
DOCBOOK_DIR="${BASE_DIR}/share/docbook"

CURL_OPTS='-C - -Os'
WGET_OPTS='-cq'

METHOD_NONE=0
METHOD_WGET=1
METHOD_CURL=2

get_url()
{
  url=$1
  case ${method} in
    ${METHOD_CURL}) ${curl} ${CURL_OPTS} "${url}" ;;
    ${METHOD_WGET}) ${wget} ${WGET_OPTS} "${url}" ;;
  esac
}

# determine method
method=${METHOD_NONE}
wget=`which wget`
[ $? -eq 0 ] && method=${METHOD_WGET}
curl=`which curl`
[ $? -eq 0 -a ${method} -eq ${METHOD_NONE} ] && method=${METHOD_CURL}
if [ ${method} -eq ${METHOD_NONE} ]; then
  echo "unable to locate wget or curl. cannot continue"
  exit 1
fi

# create download dir
mkdir -p "${DL_DIR}"

# get the docbook xml files
echo 'Docbook XML'
echo '  downloading'
cd ${DL_DIR}
get_url "${XML_URL}"
echo '  extracting'
xml_dir="${DOCBOOK_DIR}/docbook-xml/${XML_VERSION}"
rm -fr "${xml_dir}"
mkdir -p "${xml_dir}"
cd "${xml_dir}"
unzip -oq "${DL_DIR}/${XML_FILE}.zip"
cd ..
rm -f current
ln -s "${XML_VERSION}" current

# get the docbook xslt files
echo 'Docbook XSLT'
echo '  downloading'
cd ${DL_DIR}
get_url "${XSL_URL}"
echo '  extracting'
xsl_dir="${DOCBOOK_DIR}/docbook-xsl"
mkdir -p "${xsl_dir}"
cd "${xsl_dir}"
rm -fr ${XSL_VERSION}
bzip2 -dc ${DL_DIR}/${XSL_FILE}.tar.bz2 |tar xf -
mv ${XSL_FILE} ${XSL_VERSION}
rm -f current
ln -s "${XSL_VERSION}" current

{'fraction_non_alphanumeric': 0.2684026039058588, 'fraction_numerical': 0.009013520280420632, 'mean_word_length': 3.361353711790393, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2203915', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1556, 'n_words': 331}