<commit_before>#!/bin/bash -e

pushd `dirname $0`/.. > /dev/null
root=$(pwd -P)
popd > /dev/null

venv=$root/.venv
PATH=$venv/bin:$PATH

[ ! -d $venv ] && virtualenv $venv

source $venv/bin/activate

type dblatex >/dev/null 2>&1 || $venv/bin/pip install --install-option="--prefix=$venv" dblatex

source $root/ci/vars.sh

function doit {
    indir=$1
    outdir=$2
    
    cmd="$venv/bin/python $root/scripts/asciidoc-8.6.9/asciidoc.py"
    aaa=`dirname $indir/index.txt`
    bbb=`basename $aaa`
    echo "Proceesing: $bbb/index.txt"

    # txt -> html
    $cmd -o $outdir/index.html $indir/index.txt > stdout.tmp
    
    # treat asciidoc warnings as errors
    if [ -s stdout.tmp ] ;
    then
      cat stdout.tmp
      exit 1
    fi
    
    # Generate docbook xml for pdf generation
    $cmd -a lang=en -v -b docbook -d book -o $outdir/index.xml $indir/index.txt
    
    # If dblatex is available, make pdf.
    dblatex -V -T db2latex $outdir/index.xml
<commit_msg>Use easy_install instead of pip<commit_after>#!/bin/bash -e

pushd `dirname $0`/.. > /dev/null
root=$(pwd -P)
popd > /dev/null

venv=$root/.venv
PATH=$venv/bin:$PATH

[ ! -d $venv ] && virtualenv $venv

source $venv/bin/activate

type dblatex >/dev/null 2>&1 || $venv/bin/easy_install --script-dir="$venv/bin" dblatex

source $root/ci/vars.sh

function doit {
    indir=$1
    outdir=$2
    
    cmd="$venv/bin/python $root/scripts/asciidoc-8.6.9/asciidoc.py"
    aaa=`dirname $indir/index.txt`
    bbb=`basename $aaa`
    echo "Proceesing: $bbb/index.txt"

    # txt -> html
    $cmd -o $outdir/index.html $indir/index.txt > stdout.tmp
    
    # treat asciidoc warnings as errors
    if [ -s stdout.tmp ] ;
    then
      cat stdout.tmp
      exit 1
    fi
    
    # Generate docbook xml for pdf generation
    $cmd -a lang=en -v -b docbook -d book -o $outdir/index.xml $indir/index.txt
    
    # If dblatex is available, make pdf.
    dblatex -V -T db2latex $outdir/index.xml

{'fraction_non_alphanumeric': 0.14856557377049182, 'fraction_numerical': 0.010245901639344262, 'mean_word_length': 2.5062836624775584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598522', 'n_tokens_mistral': 805, 'n_tokens_neox': 786, 'n_words': 183}