<commit_before>#!/bin/bash
#
# Usage:
#   ./doc.sh <function name>

# Add the HTML shell.  A data dictionary should be readable stdin.  Does NOT
# depend on $PWD, so callers can cd.

set -o nounset
set -o errexit

to-html() {
  local out=$1
  jsont doc/html.jsont > $out
}

make-dict() {
  local body_filename=$1
  pin --title-tag=h1 $body_filename
}

# Build main docs.  Called by ./run.sh latch-demo, which calls './latch.sh
# rebuild'.
main() {
  local in=$1
  local out=$2

  local base_in=$(basename $in)  # For now, get rid of subdirs
  local body=_tmp/$base_in-body.html

  echo "Building $in -> $body -> $out"

  markdown $in >$body

  ls -al $body

  make-dict $body | to-html $out

  ls -al $out
}

# TODO: Just list the plugins/ dir?
# - need to show symlinks

plugin-types() {
  echo dot html json markdown Rplot.png treemap tar.gz txt zip
  # TODO:
  # - csv has issue with importing JSON Template; not being hermetic
  # - R generates <html>, it's not a real snippet
  # - typescript is also generates html
}

# Generate a named snippet for each plugin type.  Then join them into an HTML
# doc.

gallery-snippets() {
  local plugin_types="$1"
  local base=$2

  rm -rf $base/out  # so it's numbered from 1
  mkdir -p $base/in $base/out

  for p in $plugin_types; do
    local plugin=$PWD/plugins/$p/render 
    local input
    case $p in
      dot)
        input=$PWD/plugins/dot/testdata/cluster.dot
        ;;
      typescript)
        # disabled
        input=$PWD/plugins/typescript/testdata/typescript
        ;;
      *)
        input=$PWD/plugins/$p/testdata/tiny.$p
        ;;
    esac

    # NOTE: we are not providing a number
    local output=$p

    # plugins are written to be in the output dir.
    pushd $base/out
    $plugin $input $output
    popd >/dev/null
  done
}

print-gallery() {
  local plugin_types="$1"
  local base=$2

  cat <<EOF
<h1>webpipe Gallery</h1>

<p>Here is a list of file types and example documents.  Click through.</p>

EOF

  # Generate TOC.
  for p in $plugin_types; do
    echo "<a href=\"#$p\">$p</a> <br/>"
  done

  # Generate snippets.  TODO: should be use a <div> or something?

  for p in $plugin_types; do
    echo '<hr />'
    echo "<a id=\"$p\"></a><h3>$p</h3>"

    # TODO: alternative text?
    local path=plugins/$p/gallery.md
    if test -f $path; then
      # display on stdout
      markdown $path
    fi

    # snippet inline
    cat $base/out/$p.html
  done
}

# TODO:
# - how to get the .js to work?
# It goes ../../...
# Maybe it should be a variable, like $WEBPIPE_PLUGIN_BASE_URL.
#
# could publish it to:
# there will be:
# webpipe/plugins/
# webpipe/s/2014-04-04/1.html
# webpipe/doc/gallery/index.html
# png testdata
#   - just make a little R plot


gallery() {
  local plugin_types="$(plugin-types)"
  local base_dir=$PWD/_tmp/gallery  # absolute path

  gallery-snippets "$plugin_types" $base_dir

  local body=$base_dir/out/body.html
  local out=$base_dir/out/index.html
  print-gallery "$plugin_types" $base_dir > $body

  # NOTE: It's interesting that #anchors don't work without <html>?  At least
  # in Chrome.
  make-dict $body | to-html $out

  ls -al $base_dir/out
}

check() {
  # TODO: put this in a test?
  tidy -errors  _tmp/gallery/out/index.html
}

# NOTE: it has to be 3 levels deep
# TODO: deploy other docs
deploy-gallery() {
  set -o errexit
  # create a file in this dir with the base dir, e.g. <EMAIL>:mydir
  local base=$(cat ssh-base.txt)
  echo $base
  # Have to get all the generated dirs.  NOTE: Don't need the individual
  # snippets.  Maybe remove.
  scp -r _tmp/gallery/out/* $base/webpipe/doc/gallery
}

#
# Screencast
#

# Use the method here.  "Record my desktop", then convert using mplayer then Image Magick.
#
# http://askubuntu.com/questions/107726/how-to-create-animated-gif-images-of-a-screencast

frames() {
  local video_in=$1
  local frames_out=${2:-}
  if test -z "$frames_out"; then
    local out_dir=_tmp/frames
    mkdir -p $out_dir
    frames_out="$out_dir/$(basename $video_in)"
  fi

  # TODO: use png or gif?
  time mplayer -ao null $video_in -vo jpeg:outdir=$frames_out
}

# Took 4m 37s for a 1.4 MB ogv?  Resulted in 50 MB animated gif.
animated-gif() {
  local frames_in=$1
  local gif_out=${2:-_tmp/gif/screencast.gif}
  mkdir -p $(dirname $gif_out)
  time convert $frames_in/* $gif_out
}

# This method is down to 1MB from 50MB
optimize-gif() {
  local gif_in=$1
  local gif_out=$(echo $gif_in | sed 's/\.gif/_opt.gif/')
  set -x
  time convert $gif_in -fuzz 10% -layers Optimize $gif_out
}

"$@"
<commit_msg>Fix up gallery dir structure so output can be viewed.<commit_after>#!/bin/bash
#
# Usage:
#   ./doc.sh <function name>

# Add the HTML shell.  A data dictionary should be readable stdin.  Does NOT
# depend on $PWD, so callers can cd.

set -o nounset
set -o errexit

to-html() {
  local out=$1
  jsont doc/html.jsont > $out
}

make-dict() {
  local body_filename=$1
  pin --title-tag=h1 $body_filename
}

# Build main docs.  Called by ./run.sh latch-demo, which calls './latch.sh
# rebuild'.
main() {
  local in=$1
  local out=$2

  local base_in=$(basename $in)  # For now, get rid of subdirs
  local body=_tmp/$base_in-body.html

  echo "Building $in -> $body -> $out"

  markdown $in >$body

  ls -al $body

  make-dict $body | to-html $out

  ls -al $out
}

# TODO: Just list the plugins/ dir?
# - need to show symlinks

plugin-types() {
  echo dot html json markdown Rplot.png treemap tar.gz txt zip
  # TODO:
  # - csv has issue with importing JSON Template; not being hermetic
  # - R generates <html>, it's not a real snippet
  # - typescript is also generates html
}

# Generate a named snippet for each plugin type.  Then join them into an HTML
# doc.

gallery-snippets() {
  local plugin_types="$1"
  local out_dir=$2

  rm -rf $out_dir
  mkdir -p $out_dir

  for p in $plugin_types; do
    local plugin=$PWD/plugins/$p/render 
    local input
    case $p in
      dot)
        input=$PWD/plugins/dot/testdata/cluster.dot
        ;;
      typescript)
        # disabled
        input=$PWD/plugins/typescript/testdata/typescript
        ;;
      *)
        input=$PWD/plugins/$p/testdata/tiny.$p
        ;;
    esac

    # NOTE: we are not providing a number
    local output=$p

    # plugins are written to be in the output dir.
    pushd $out_dir
    $plugin $input $output
    popd >/dev/null
  done
}

print-gallery() {
  local plugin_types="$1"
  local base_dir=$2

  cat <<EOF
<h1>webpipe Gallery</h1>

<p>Here is a list of file types and example documents.  There is a JavaScript
visualization behind some links.</p>
EOF

  # Generate TOC.
  for p in $plugin_types; do
    echo "<a href=\"#$p\">$p</a> <br/>"
  done

  # Generate snippets.  TODO: should be use a <div> or something?

  for p in $plugin_types; do
    echo '<hr />'
    echo "<a id=\"$p\"></a><h3>$p</h3>"

    # TODO: alternative text?
    local path=plugins/$p/gallery.md
    if test -f $path; then
      # display on stdout
      markdown $path
    fi

    # snippet inline
    cat $base_dir/$p.html
  done
}

# TODO:
# - how to get the .js to work?
# It goes ../../...
# Maybe it should be a variable, like $WEBPIPE_PLUGIN_BASE_URL.
#
# could publish it to:
# there will be:
# webpipe/plugins/
# webpipe/s/2014-04-04/1.html
# webpipe/doc/gallery/index.html
# png testdata
#   - just make a little R plot


gallery() {
  local plugin_types="$(plugin-types)"
  local base_dir=$PWD/_tmp/gallery  # absolute path

  gallery-snippets "$plugin_types" $base_dir

  local body=$base_dir/body.html
  local out=$base_dir/index.html
  print-gallery "$plugin_types" $base_dir > $body

  # NOTE: It's interesting that #anchors don't work without <html>?  At least
  # in Chrome.
  make-dict $body | to-html $out

  # The gallery needs to link to static assets.  TODO: Should be _tmp/doc?
  ln -v -s $PWD/plugins _tmp/plugins

  ls -al $base_dir
}

check() {
  # TODO: put this in a test?
  tidy -errors  _tmp/gallery/out/index.html
}

# NOTE: it has to be 3 levels deep
# TODO: deploy other docs
deploy-gallery() {
  set -o errexit
  # create a file in this dir with the base dir, e.g. <EMAIL>:mydir
  local base=$(cat ssh-base.txt)
  echo $base
  # Have to get all the generated dirs.  NOTE: Don't need the individual
  # snippets.  Maybe remove.
  scp -r _tmp/gallery/out/* $base/webpipe/doc/gallery
}

#
# Screencast
#

# Use the method here.  "Record my desktop", then convert using mplayer then Image Magick.
#
# http://askubuntu.com/questions/107726/how-to-create-animated-gif-images-of-a-screencast

frames() {
  local video_in=$1
  local frames_out=${2:-}
  if test -z "$frames_out"; then
    local out_dir=_tmp/frames
    mkdir -p $out_dir
    frames_out="$out_dir/$(basename $video_in)"
  fi

  # TODO: use png or gif?
  time mplayer -ao null $video_in -vo jpeg:outdir=$frames_out
}

# Took 4m 37s for a 1.4 MB ogv?  Resulted in 50 MB animated gif.
animated-gif() {
  local frames_in=$1
  local gif_out=${2:-_tmp/gif/screencast.gif}
  mkdir -p $(dirname $gif_out)
  time convert $frames_in/* $gif_out
}

# This method is down to 1MB from 50MB
optimize-gif() {
  local gif_in=$1
  local gif_out=$(echo $gif_in | sed 's/\.gif/_opt.gif/')
  set -x
  time convert $gif_in -fuzz 10% -layers Optimize $gif_out
}

"$@"

{'fraction_non_alphanumeric': 0.1265959748972084, 'fraction_numerical': 0.010062756979008873, 'mean_word_length': 2.841645885286783, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6654335', 'n_tokens_mistral': 3507, 'n_tokens_neox': 3315, 'n_words': 1066}