<commit_before>#!/bin/bash
#
# Demo of RAPPOR.  Automating Python and R scripts.  See README.
#
# Usage:
#   ./demo.sh <function name>
#
# End to end demo for 3 distributions:
#
#   $ ./demo.sh run
#
# Run demo for just one distribution (no HTML output):
#
#   $ ./demo.sh run-dist [exp|gauss|unif]
#
# (This takes a minute or so)

set -o nounset
set -o pipefail
set -o errexit

readonly THIS_DIR=$(dirname $0)
readonly REPO_ROOT=$THIS_DIR
readonly CLIENT_DIR=$REPO_ROOT/client/python

#
# Utility functions
#

banner() {
  echo
  echo "----- $@"
  echo
}

log() {
  echo 1>&2 "$@"
}

die() {
  log "$0: $@"
  exit 1
}

#
# Semi-automated demos
#

# This generates the simulated input s1 .. s<n> with 3 different distributions.
gen-sim-input() {
  local dist=$1
  local num_clients=$2

  local flag=''
  case $dist in
    exp)
      flag=-e
      ;;
    gauss)
      flag=-g
      ;;
    unif)
      flag=-u
      ;;
    *)
      die "Invalid distribution '$dist'"
  esac

  mkdir -p _tmp

  # Simulating 10,000 clients runs reasonably fast but the results look poor.
  # 100,000 is slow but looks better.
  # 50 different client values are easier to plot (default is 100)
  time tests/gen_sim_input.py $flag \
    -n $num_clients \
    -r 50 \
    -o _tmp/$dist.csv
}

# Do the RAPPOR transformation on our simulated input.
rappor-sim() {
  local dist=$1
  shift
  PYTHONPATH=$CLIENT_DIR time $REPO_ROOT/tests/rappor_sim.py \
    -i _tmp/$dist.csv \
    "$@"
    #-s 0  # deterministic seed
}

hash-candidates() {
  local dist=$1
  shift
  local out=_tmp/${dist}_map.csv
  PYTHONPATH=$CLIENT_DIR time tests/hash_candidates.py \
    < _tmp/${dist}_candidates.txt \
    > $out
  log "Wrote $out"
}

sum-bits() {
  local dist=$1
  shift
  local out=_tmp/${dist}_counts.csv
  tests/sum_bits.py < _tmp/${dist}_out.csv > $out
  log "Wrote $out"
}

# Like rappor-sim, but run it through the Python profiler.
rappor-sim-profile() {
  local dist=$1
  shift

  export PYTHONPATH=$CLIENT_DIR
  # For now, just dump it to a text file.  Sort by cumulative time.
  time python -m cProfile -s cumulative \
    tests/rappor_sim.py \
    -i _tmp/$dist.csv \
    "$@" \
    | tee _tmp/profile.txt
}

# Analyze output of Python client library.
analyze() {
  local dist=$1
  local title=$2
  local prefix=_tmp/$dist

  local out_dir=_tmp/${dist}_report
  mkdir -p $out_dir

  # The shebang on analyze.R is /usr/bin/Rscript.  With some Linux distros
  # (Ubuntu), you often need to compile your own R to get say R 3.0 instead of
  # 2.14.  In that case, do something like:
  #
  # export R_SCRIPT=/usr/local/bin/Rscript<commit_msg>Cosmetic change: Keep the two variants of rappor-sim together.<commit_after>#!/bin/bash
#
# Demo of RAPPOR.  Automating Python and R scripts.  See README.
#
# Usage:
#   ./demo.sh <function name>
#
# End to end demo for 3 distributions:
#
#   $ ./demo.sh run
#
# Run demo for just one distribution (no HTML output):
#
#   $ ./demo.sh run-dist [exp|gauss|unif]
#
# (This takes a minute or so)

set -o nounset
set -o pipefail
set -o errexit

readonly THIS_DIR=$(dirname $0)
readonly REPO_ROOT=$THIS_DIR
readonly CLIENT_DIR=$REPO_ROOT/client/python

#
# Utility functions
#

banner() {
  echo
  echo "----- $@"
  echo
}

log() {
  echo 1>&2 "$@"
}

die() {
  log "$0: $@"
  exit 1
}

#
# Semi-automated demos
#

# This generates the simulated input s1 .. s<n> with 3 different distributions.
gen-sim-input() {
  local dist=$1
  local num_clients=$2

  local flag=''
  case $dist in
    exp)
      flag=-e
      ;;
    gauss)
      flag=-g
      ;;
    unif)
      flag=-u
      ;;
    *)
      die "Invalid distribution '$dist'"
  esac

  mkdir -p _tmp

  # Simulating 10,000 clients runs reasonably fast but the results look poor.
  # 100,000 is slow but looks better.
  # 50 different client values are easier to plot (default is 100)
  time tests/gen_sim_input.py $flag \
    -n $num_clients \
    -r 50 \
    -o _tmp/$dist.csv
}

# Do the RAPPOR transformation on our simulated input.
rappor-sim() {
  local dist=$1
  shift
  PYTHONPATH=$CLIENT_DIR time \
    tests/rappor_sim.py \
    -i _tmp/$dist.csv \
    "$@"
    #-s 0  # deterministic seed
}

# Like rappor-sim, but run it through the Python profiler.
rappor-sim-profile() {
  local dist=$1
  shift

  export PYTHONPATH=$CLIENT_DIR
  # For now, just dump it to a text file.  Sort by cumulative time.
  time python -m cProfile -s cumulative \
    tests/rappor_sim.py \
    -i _tmp/$dist.csv \
    "$@" \
    | tee _tmp/profile.txt
}

hash-candidates() {
  local dist=$1
  shift
  local out=_tmp/${dist}_map.csv
  PYTHONPATH=$CLIENT_DIR time tests/hash_candidates.py \
    < _tmp/${dist}_candidates.txt \
    > $out
  log "Wrote $out"
}

sum-bits() {
  local dist=$1
  shift
  local out=_tmp/${dist}_counts.csv
  tests/sum_bits.py < _tmp/${dist}_out.csv > $out
  log "Wrote $out"
}

# Analyze output of Python client library.
analyze() {
  local dist=$1
  local title=$2
  local prefix=_tmp/$dist

  local out_dir=_tmp/${dist}_report
  mkdir -p $out_dir

  # The shebang on analyze.R is /usr/bin/Rscript.  With some Linux distros
  # (Ubuntu), you often need to compile your own R to get say R 3.0 instead of
  # 2.14.  In that case, do something like:
  #
  # export R_SCRIPT=/usr/local/bin/Rscript
{'fraction_non_alphanumeric': 0.12647170527914925, 'fraction_numerical': 0.015191796429927839, 'mean_word_length': 2.7248939179632248, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3758184', 'n_tokens_mistral': 2049, 'n_tokens_neox': 1941, 'n_words': 577}