<commit_before>#!/usr/bin/env bash
#
# TODO - Add --verbose or --silent option.
#      - Pass a directory of ebook files, and use `parallel` (if installed) to process batches.
#      - Add --server option to pick beta or production server.

if [ $# -ne 2 ]; then
    echo -e "Upgrade and/or fix an EPUB file using the Bookalope cloud service.\nMissing argument(s)\nUsage: $0 token epub"
    exit 1
fi

TOKEN=$1
EBOOKFILE=$2
TMPDIR=$(mktemp -d)

# Make sure that the ebook file actually exists.
if [ ! -f "$EBOOKFILE" ]; then
    echo "Ebook file $EBOOKFILE does not exist, exiting"
    exit 1
fi

# Check that Python 3 is available.
if [ ! `builtin type -p python3` ]; then
    echo "This script requires Python 3 to be installed, exiting"
    exit 1
fi

# Check that the Bookalope authentication token has the right format.
if [[ ! $TOKEN =~ ^[0-9a-fA-F]{32}$ ]]; then
    echo "Malformed Bookalope API token, exiting"
    exit 1
fi

# The Bookalope server.
APIHOST="https://beta.bookalope.net"
echo "Talking to Bookalope server $APIHOST"

# Separate path, filename, and extension of the document.
EBOOKPATH=$(dirname "$EBOOKFILE")
EBOOKNAME=$(basename "$EBOOKFILE")
EBOOKBASE="${EBOOKNAME%.*}"

# Wait for a given number of seconds while showing a spinner.
function wait() {
    local COUNT=$1
    while ((COUNT--)); do
        for SPIN in '-' '\' '|' '/'; do
            echo -en "Waiting for bookflow to finish $SPIN \r"
            sleep 0.25
        done
    done
}

# Use httpie to talk to the Bookalope server.
if [ `builtin type -p http` ]; then

    # Check that the Bookalope token authenticates correctly with the server.
    if [ ! `http --headers --auth $TOKEN: GET $APIHOST/api/profile | grep HTTP | cut -d ' ' -f 2` == "200" ]; then
        echo "Wrong Bookalope API token, exiting"
        exit 1
    fi

    # Create a new book, and use the book's initial bookflow.
    echo "Creating new Book..."
    read -r BOOKID BOOKFLOWID <<< `http --json --print=b --auth $TOKEN: POST $APIHOST/api/books name="$EBOOKBASE" | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['book']['id'], obj['book']['bookflows'][0]['id']);"`
    echo "Done, book=$BOOKID, bookflow=$BOOKFLOWID"

    # Upload the ebook file which automatically ingests its content and styling. Passing the `ignore_analysis`
    # argument here tells Bookalope to ignore the AI-assisted semantic structuring of the ebook, and instead
    # carry through the ebook's visual styles (AKA WYSIWYG conversion). The result is a flat and unstructured
    # ebook, but it is at least a valid EPUB3 file. So make sure you know what you're doing here.
    echo "Uploading and ingesting ebook file: $EBOOKNAME"
    base64 "$EBOOKFILE" > "$TMPDIR/$EBOOKNAME.base64"
    http --json --auth $TOKEN: POST $APIHOST/api/bookflows/$BOOKFLOWID/files/document file=@"$TMPDIR/$EBOOKNAME.base64" filename="$EBOOKNAME" filetype=epub ignore_analysis=true

    # Wait until the bookflow's step changes from 'processing' to 'convert', thus indicating that Bookalope
    # has finished noodling through the ebook.
    while true; do
        wait 5
        STEP=`http --json --print=b --auth $TOKEN: GET $APIHOST/api/bookflows/$BOOKFLOWID | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['bookflow']['step']);"`
        if [ "$STEP" = "convert" ]; then
            break
        fi
        if [ "$STEP" = "processing_failed" ]; then
            echo "Bookalope failed to ingest the ebook, exiting"
            exit 1
        fi
    done
    echo "Waiting for bookflow to finish, done!"

    # Convert the ingested ebook file to EPUB3 and download it.
    # Regarding < /dev/tty see: https://github.com/jakubroztocil/httpie/issues/150#issuecomment-21419373
    echo "Converting and downloading EPUB3 format..."
    DOWNLOAD_URL=`http --auth $TOKEN: POST $APIHOST/api/bookflows/$BOOKFLOWID/convert format=epub3 version=final < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['download_url'])"`
    while true; do
        wait 5
        STATUS=`http --auth $TOKEN: GET $DOWNLOAD_URL/status < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['status']);"`
        case "$STATUS" in
        "processing")
            ;;
        "failed")
            echo "Bookalope failed to convert the ebook, exiting"
            exit 1
            ;;
        "ok")
            echo "Waiting for bookflow to finish, done!"
            break
            ;;
        esac
    done
    http --download --auth $TOKEN: GET $DOWNLOAD_URL > /dev/tty
    mv $BOOKFLOWID.epub "${EBOOKFILE%.*}-$BOOKFLOWID.epub"
    echo "Saved converted ebook to file ${EBOOKFILE%.*}-$BOOKFLOWID.epub"

    # Delete the book and its bookflow.
    echo "Deleting book and bookflows..."
    http --auth $TOKEN: DELETE $APIHOST/api/books/$BOOKID
    echo "Done"

else

    # Use curl to talk to the Bookalope server.
    if [ `builtin type -p curl` ]; then

        # Check that the Bookalope token authenticates correctly with the server.
        if [ ! `curl --user $TOKEN: --request GET -s -D - -o /dev/null $APIHOST/api/profile | grep HTTP | cut -d ' ' -f 2` == "200" ]; then
            echo "Wrong Bookalope API token, exiting"
            exit 1
        fi

        # Create a new book, and use the book's initial bookflow.
        echo "Creating new Book..."
        read -r BOOKID BOOKFLOWID <<< `curl --user $TOKEN: --header "Content-Type: application/json" --data '{"name":"'$EBOOKBASE'"}' --request POST $APIHOST/api/books | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['book']['id'], obj['book']['bookflows'][0]['id']);"`
        echo "Done, book=$BOOKID, bookflow=$BOOKFLOWID"

        # Upload the ebook file which automatically ingests its content and styling. Passing the `ignore_analysis`
        # argument here tells Bookalope to ignore the AI-assisted semantic structuring of the ebook, and instead
        # carry through the ebook's visual styles (AKA WYSIWYG conversion). The result is a flat and unstructured
        # ebook, but it is at least a valid EPUB3 file. So make sure you know what you're doing here.
        echo "Uploading and ingesting ebook: $EBOOKNAME"
        echo '{"filetype":"epub", "filename":"'$EBOOKNAME'", "ignore_analysis":"true", "file":"' > "$TMPDIR/$EBOOKNAME.json"
        base64 "$EBOOKFILE" >> "$TMPDIR/$EBOOKNAME.json"
        echo '"}' >> "$TMPDIR/$EBOOKNAME.json"
        curl --user $TOKEN: --header "Content-Type: application/json" --data @"$TMPDIR/$EBOOKNAME.json" --request POST $APIHOST/api/bookflows/$BOOKFLOWID/files/document

        # Wait until the bookflow's step changes from 'processing' to 'convert', thus indicating that Bookalope
        # has finished noodling through the ebook.
        while true; do
            wait 5
            STEP=`curl --user $TOKEN: --header "Content-Type: application/json" --request GET $APIHOST/api/bookflows/$BOOKFLOWID | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['bookflow']['step']);"`
            if [ "$STEP" = "convert" ]; then
                break
            fi
            if [ "$STEP" = "processing_failed" ]; then
                echo "Bookalope failed to ingest the ebook, exiting"
                exit 1
            fi
        done
        echo "Waiting for bookflow to finish, done!"

        # Convert the ingested ebook file to EPUB3 and download it.
        # Regarding < /dev/tty see: https://github.com/jakubroztocil/httpie/issues/150#issuecomment-21419373
        echo "Converting and downloading books..."
        DOWNLOAD_URL=`curl --user $TOKEN: --header "Content-Type: application/json" --data '{"format":"epub3", "version":"final"}' --request POST $APIHOST/api/bookflows/$BOOKFLOWID/convert < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['download_url'])"`
        while true; do
            wait 5
            STATUS=`curl --user $TOKEN: --header "Content-Type: application/json" --request GET $DOWNLOAD_URL/status < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['status']);"`
            case "$STATUS" in
            "processing")
                ;;
            "failed")
                echo "Bookalope failed to convert the ebook, exiting"
                exit 1
                ;;
            "ok")
                echo "Waiting for bookflow to finish, done!"
                break
                ;;
            esac
        done
        curl --user $TOKEN: --remote-name --remote-header-name --request GET $DOWNLOAD_URL > /dev/tty
        mv $BOOKFLOWID.epub "${EBOOKFILE%.*}-$BOOKFLOWID.epub"
        echo "Saved converted ebook to file ${EBOOKFILE%.*}-$BOOKFLOWID.epub"

        # Delete the book and its bookflow.
        echo "Deleting book and bookflows..."
        curl --user $TOKEN: --request DELETE $APIHOST/api/books/$BOOKID
        echo "Done"

    else
        echo "Unable to find http or curl command, exiting"
        exit 1
    fi
fi
rm -fr $TMPDIR
exit 0
<commit_msg>Silence httpie, curl and make progress output more readable.<commit_after>#!/usr/bin/env bash
#
# TODO - Add --verbose or --silent option.
#      - Pass a directory of ebook files, and use `parallel` (if installed) to process batches.
#      - Add --server option to pick beta or production server.

if [ $# -ne 2 ]; then
    echo -e "Upgrade and/or fix an EPUB file using the Bookalope cloud service.\nMissing argument(s)\nUsage: $0 token epub"
    exit 1
fi

TOKEN=$1
EBOOKFILE=$2
TMPDIR=$(mktemp -d)

# Make sure that the ebook file actually exists.
if [ ! -f "$EBOOKFILE" ]; then
    echo "Ebook file $EBOOKFILE does not exist, exiting"
    exit 1
fi

# Check that Python 3 is available.
if [ ! `builtin type -p python3` ]; then
    echo "This script requires Python 3 to be installed, exiting"
    exit 1
fi

# Check that the Bookalope authentication token has the right format.
if [[ ! $TOKEN =~ ^[0-9a-fA-F]{32}$ ]]; then
    echo "Malformed Bookalope API token, exiting"
    exit 1
fi

# The Bookalope server.
APIHOST="https://beta.bookalope.net"
echo "Talking to Bookalope server $APIHOST"

# Separate path, filename, and extension of the document.
EBOOKPATH=$(dirname "$EBOOKFILE")
EBOOKNAME=$(basename "$EBOOKFILE")
EBOOKBASE="${EBOOKNAME%.*}"

# Wait for a given number of seconds while showing a spinner.
function wait() {
    local COUNT=$1
    while ((COUNT--)); do
        for SPIN in '-' '\' '|' '/'; do
            echo -en "Waiting for Bookflow to finish $SPIN \r"
            sleep 0.25
        done
    done
}

# Use httpie to talk to the Bookalope server.
if [ `builtin type -p http` ]; then

    # Check that the Bookalope token authenticates correctly with the server.
    if [ ! `http --headers --auth $TOKEN: GET $APIHOST/api/profile | grep HTTP | cut -d ' ' -f 2` == "200" ]; then
        echo "Wrong Bookalope API token, exiting"
        exit 1
    fi

    # Create a new book, and use the book's initial bookflow.
    echo "Creating new Book..."
    read -r BOOKID BOOKFLOWID <<< `http --ignore-stdin --json --print=b --auth $TOKEN: POST $APIHOST/api/books name="$EBOOKBASE" | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['book']['id'], obj['book']['bookflows'][0]['id']);"`
    echo "Done, Book id=$BOOKID, Bookflow id=$BOOKFLOWID"

    # Upload the ebook file which automatically ingests its content and styling. Passing the `ignore_analysis`
    # argument here tells Bookalope to ignore the AI-assisted semantic structuring of the ebook, and instead
    # carry through the ebook's visual styles (AKA WYSIWYG conversion). The result is a flat and unstructured
    # ebook, but it is at least a valid EPUB3 file. So make sure you know what you're doing here.
    echo "Uploading and ingesting ebook file: $EBOOKNAME"
    base64 "$EBOOKFILE" > "$TMPDIR/$EBOOKNAME.base64"
    http --ignore-stdin --json --print= --auth $TOKEN: POST $APIHOST/api/bookflows/$BOOKFLOWID/files/document file=@"$TMPDIR/$EBOOKNAME.base64" filename="$EBOOKNAME" filetype=epub ignore_analysis=true

    # Wait until the bookflow's step changes from 'processing' to 'convert', thus indicating that Bookalope
    # has finished noodling through the ebook.
    while true; do
        wait 5
        STEP=`http --ignore-stdin --json --print=b --auth $TOKEN: GET $APIHOST/api/bookflows/$BOOKFLOWID | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['bookflow']['step']);"`
        if [ "$STEP" = "convert" ]; then
            echo "Waiting for Bookflow to finish, done!"
            break
        fi
        if [ "$STEP" = "processing_failed" ]; then
            echo "Bookalope failed to ingest the ebook, exiting"
            exit 1
        fi
    done

    # Convert the ingested ebook file to EPUB3 and download it.
    # Regarding < /dev/tty see: https://github.com/jakubroztocil/httpie/issues/150#issuecomment-21419373
    echo "Converting to EPUB3 format and downloading ebook file..."
    DOWNLOAD_URL=`http --auth $TOKEN: POST $APIHOST/api/bookflows/$BOOKFLOWID/convert format=epub3 version=final < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['download_url'])"`
    while true; do
        wait 5
        STATUS=`http --auth $TOKEN: GET $DOWNLOAD_URL/status < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['status']);"`
        case "$STATUS" in
        "processing")
            ;;
        "failed")
            echo "Bookalope failed to convert the ebook, exiting"
            exit 1
            ;;
        "ok")
            echo "Waiting for Bookflow to finish, done!"
            break
            ;;
        esac
    done
    http --download --ignore-stdin --print= --auth $TOKEN: GET $DOWNLOAD_URL > /dev/tty
    mv $BOOKFLOWID.epub "${EBOOKFILE%.*}-$BOOKFLOWID.epub"
    echo "Saved converted ebook to file ${EBOOKFILE%.*}-$BOOKFLOWID.epub"

    # Delete the book and its bookflow.
    echo "Deleting Book and Bookflow..."
    http --ignore-stdin --print= --auth $TOKEN: DELETE $APIHOST/api/books/$BOOKID
    echo "Done"

else

    # Use curl to talk to the Bookalope server.
    if [ `builtin type -p curl` ]; then

        # Check that the Bookalope token authenticates correctly with the server.
        if [ ! `curl --silent --show-error --user $TOKEN: --request GET -s -D - -o /dev/null $APIHOST/api/profile | grep HTTP | cut -d ' ' -f 2` == "200" ]; then
            echo "Wrong Bookalope API token, exiting"
            exit 1
        fi

        # Create a new book, and use the book's initial bookflow.
        echo "Creating new Book..."
        read -r BOOKID BOOKFLOWID <<< `curl --silent --show-error --user $TOKEN: --header "Content-Type: application/json" --data '{"name":"'$EBOOKBASE'"}' --request POST $APIHOST/api/books | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['book']['id'], obj['book']['bookflows'][0]['id']);"`
        echo "Done, Book id=$BOOKID, Bookflow id=$BOOKFLOWID"

        # Upload the ebook file which automatically ingests its content and styling. Passing the `ignore_analysis`
        # argument here tells Bookalope to ignore the AI-assisted semantic structuring of the ebook, and instead
        # carry through the ebook's visual styles (AKA WYSIWYG conversion). The result is a flat and unstructured
        # ebook, but it is at least a valid EPUB3 file. So make sure you know what you're doing here.
        echo "Uploading and ingesting ebook file: $EBOOKNAME"
        echo '{"filetype":"epub", "filename":"'$EBOOKNAME'", "ignore_analysis":"true", "file":"' > "$TMPDIR/$EBOOKNAME.json"
        base64 "$EBOOKFILE" >> "$TMPDIR/$EBOOKNAME.json"
        echo '"}' >> "$TMPDIR/$EBOOKNAME.json"
        curl --silent --show-error --user $TOKEN: --header "Content-Type: application/json" --data @"$TMPDIR/$EBOOKNAME.json" --request POST $APIHOST/api/bookflows/$BOOKFLOWID/files/document

        # Wait until the bookflow's step changes from 'processing' to 'convert', thus indicating that Bookalope
        # has finished noodling through the ebook.
        while true; do
            wait 5
            STEP=`curl --silent --show-error --user $TOKEN: --header "Content-Type: application/json" --request GET $APIHOST/api/bookflows/$BOOKFLOWID | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['bookflow']['step']);"`
            if [ "$STEP" = "convert" ]; then
                echo "Waiting for Bookflow to finish, done!"
                break
            fi
            if [ "$STEP" = "processing_failed" ]; then
                echo "Bookalope failed to ingest the ebook, exiting"
                exit 1
            fi
        done

        # Convert the ingested ebook file to EPUB3 and download it.
        # Regarding < /dev/tty see: https://github.com/jakubroztocil/httpie/issues/150#issuecomment-21419373
        echo "Converting to EPUB3 format and downloading ebook file..."
        DOWNLOAD_URL=`curl --silent --show-error --user $TOKEN: --header "Content-Type: application/json" --data '{"format":"epub3", "version":"final"}' --request POST $APIHOST/api/bookflows/$BOOKFLOWID/convert < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['download_url'])"`
        while true; do
            wait 5
            STATUS=`curl --silent --show-error --user $TOKEN: --header "Content-Type: application/json" --request GET $DOWNLOAD_URL/status < /dev/tty | python3 -c "import json,sys;obj=json.load(sys.stdin);print(obj['status']);"`
            case "$STATUS" in
            "processing")
                ;;
            "failed")
                echo "Bookalope failed to convert the ebook, exiting"
                exit 1
                ;;
            "ok")
                echo "Waiting for Bookflow to finish, done!"
                break
                ;;
            esac
        done
        curl --silent --show-error --user $TOKEN: --remote-name --remote-header-name --request GET $DOWNLOAD_URL > /dev/tty
        mv $BOOKFLOWID.epub "${EBOOKFILE%.*}-$BOOKFLOWID.epub"
        echo "Saved converted ebook to file ${EBOOKFILE%.*}-$BOOKFLOWID.epub"

        # Delete the book and its bookflow.
        echo "Deleting Book and Bookflow..."
        curl --silent --show-error --user $TOKEN: --request DELETE $APIHOST/api/books/$BOOKID
        echo "Done"

    else
        echo "Unable to find http or curl command, exiting"
        exit 1
    fi
fi
rm -fr $TMPDIR
exit 0

{'fraction_non_alphanumeric': 0.12642114997552087, 'fraction_numerical': 0.009410868737420443, 'mean_word_length': 2.613917829762139, 'pattern_counts': {'":': 14, '<': 27, '<?xml version=': 0, '>': 19, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1161582', 'n_tokens_mistral': 5918, 'n_tokens_neox': 5646, 'n_words': 1876}