<commit_before>#!/bin/bash

set -beEu -o pipefail

# Script to generate the sphinx documentation
# Run from inside schemas/sphinx

# Expects sphinx and pypandoc to be installed

# Meant to be run from the Maven build, so set our cwd relative to the top-level dir
cd doc

# Create a home for all the merged .rst files: the written ones and the generated ones

GENERATED_DOCS_LOC=../target/generated-docs
RST_LOC=$GENERATED_DOCS_LOC/rst
#RST_SCHEMAS_LOC=$RST_LOC/schemas
RST_SCHEMAS_LOC=$RST_LOC
# HTML_LOC is relative to $GENERATED_DOCS_LOC
HTML_LOC=merged
mkdir -p $RST_SCHEMAS_LOC

# Generate AVPR files

# Are the Avro tools installed?
AVRO_TOOLS_LOC=../target/avro-tools
if [ ! -f $AVRO_TOOLS_LOC/avro-tools.jar ]
then
    # Download them if not
    echo -n Downloading Avro tools...
    mkdir -p $AVRO_TOOLS_LOC
    curl -s -o $AVRO_TOOLS_LOC/avro-tools.jar http://www.us.apache.org/dist/avro/avro-1.7.7/java/avro-tools-1.7.7.jar
    echo " done."
fi
<commit_msg>Switched back to putting generated Avro docs in the "schemas" subdirectory.  This avoids possible name collisions between written and generated file names.<commit_after>#!/bin/bash

set -beEu -o pipefail

# Script to generate the sphinx documentation
# Run from inside schemas/sphinx

# Expects sphinx and pypandoc to be installed

# Meant to be run from the Maven build, so set our cwd relative to the top-level dir
cd doc

GENERATED_DOCS_LOC=../target/generated-docs
RST_LOC=$GENERATED_DOCS_LOC/rst
RST_SCHEMAS_LOC=$RST_LOC/schemas
# HTML_LOC is where the final output goes.  It's relative to $GENERATED_DOCS_LOC.
HTML_LOC=merged

# Create a home for all the merged .rst files: the written ones and the generated ones

mkdir -p $RST_SCHEMAS_LOC

# Generate AVPR files

# Are the Avro tools installed?
AVRO_TOOLS_LOC=../target/avro-tools
if [ ! -f $AVRO_TOOLS_LOC/avro-tools.jar ]
then
    # Download them if not
    echo -n Downloading Avro tools...
    mkdir -p $AVRO_TOOLS_LOC
    curl -s -o $AVRO_TOOLS_LOC/avro-tools.jar http://www.us.apache.org/dist/avro/avro-1.7.7/java/avro-tools-1.7.7.jar
    echo " done."
fi

{'fraction_non_alphanumeric': 0.08912314326784859, 'fraction_numerical': 0.005749880210828941, 'mean_word_length': 3.878504672897196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7133495', 'n_tokens_mistral': 781, 'n_tokens_neox': 755, 'n_words': 232}