<commit_before>#!/bin/bash
#
# Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications.
# Designed for optimal compatibility with the Keyman Suite.
#

EMBEDDED_WORKER=embedded_worker.js

# Build the worker and the main script.
build ( ) {
  # Build worker first; the main file depends on it.
  # Then wrap the worker; Then build the main file.
  build-worker && wrap_worker && build-main
}

# Builds the top-level JavaScript file (the second stage of compilation)
build-main () {
  npm run tsc -- -p ./tsconfig.json || fail "Could not build top-level JavaScript file."
}

# Builds the inner JavaScript worker (the first stage of compilation).
# This script must be wrapped.
build-worker () {
  npm run tsc -- -p ./worker/tsconfig.json || fail "Could not build worker."
}

# Creates embedded_worker.js. Must be run after the worker is built for the
# first time
wrap_worker ( ) {
  echo "> wrap_worker_code LMLayerWorkerCode ./worker/index.js > ${EMBEDDED_WORKER}"
  wrap_worker_code LMLayerWorkerCode ./worker/index.js > "${EMBEDDED_WORKER}"
}

# A nice, extensible method for -clean operations.  Add to this as necessary.
clean ( ) {
  rm -f "${EMBEDDED_WORKER}"
  if [ $? -ne 0 ]; then
    fail "Failed to erase the prior build."
  fi
}

display_usage ( ) {
  echo "Usage: $0 [-clean] [-test | -tdd]"
  echo "       $0 -help"
  echo
  echo "  -clean              to erase pre-existing build products before a re-build"
  echo "  -help               displays this screen and exits"
  echo "  -tdd                skips dependency updates, builds, then runs unit tests only"
  echo "  -test               runs unit and integration tests after building"
}

# Prints a nice, common error message.
fail ( ) {
  local ERROR_RED
  local NORMAL
  ERROR_RED="$(tput setaf 1)"
  NORMAL="$(tput sgr0)"
  FAILURE_MSG="$1"
  if [[ "$FAILURE_MSG" == "" ]]; then
    FAILURE_MSG="Unknown failure."
  fi
  echo "$0: ${ERROR_RED}$FAILURE_MSG${NORMAL}"
  exit 1
}

# Wraps JavaScript code in a way that can be embedded in a worker.
# To get the inner source code, include the file generated by this function,
# then use name.toString() where `name` is the name passed into this
# function.
wrap_worker_code ( ) {
  name="$1"
  js="$2"
  echo "// Autogenerated code. Do not modify!"
  printf "function %s () {\n" "${name}"
  cat "${js}"
  printf "\n}\n"
}

################################ Main script ################################

run_tests=0
fetch_deps=1
unit_tests_only=0

# Process command-line arguments
while [[ $# -gt 0 ]] ; do
  key="$1"
  case $key in
    -clean)
      clean
      ;;
    -help|-h)
      display_usage
      exit
      ;;
    -test)
      run_tests=1
      ;;<commit_msg>Use dashes to make names consistent.<commit_after>#!/bin/bash
#
# Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications.
# Designed for optimal compatibility with the Keyman Suite.
#

EMBEDDED_WORKER=embedded_worker.js

# Build the worker and the main script.
build ( ) {
  # Build worker first; the main file depends on it.
  # Then wrap the worker; Then build the main file.
  build-worker && wrap-worker && build-main
}

# Builds the top-level JavaScript file (the second stage of compilation)
build-main () {
  npm run tsc -- -p ./tsconfig.json || fail "Could not build top-level JavaScript file."
}

# Builds the inner JavaScript worker (the first stage of compilation).
# This script must be wrapped.
build-worker () {
  npm run tsc -- -p ./worker/tsconfig.json || fail "Could not build worker."
}

# Creates embedded_worker.js. Must be run after the worker is built for the
# first time
wrap-worker ( ) {
  echo "> wrap-worker-code LMLayerWorkerCode ./worker/index.js > ${EMBEDDED_WORKER}"
  wrap-worker-code LMLayerWorkerCode ./worker/index.js > "${EMBEDDED_WORKER}"
}

# A nice, extensible method for -clean operations.  Add to this as necessary.
clean ( ) {
  rm -f "${EMBEDDED_WORKER}"
  if [ $? -ne 0 ]; then
    fail "Failed to erase the prior build."
  fi
}

display_usage ( ) {
  echo "Usage: $0 [-clean] [-test | -tdd]"
  echo "       $0 -help"
  echo
  echo "  -clean              to erase pre-existing build products before a re-build"
  echo "  -help               displays this screen and exits"
  echo "  -tdd                skips dependency updates, builds, then runs unit tests only"
  echo "  -test               runs unit and integration tests after building"
}

# Prints a nice, common error message.
fail ( ) {
  # TODO: source shellHelperFunctions.sh
  local ERROR_RED
  local NORMAL
  ERROR_RED="$(tput setaf 1)"
  NORMAL="$(tput sgr0)"
  FAILURE_MSG="$1"
  if [[ "$FAILURE_MSG" == "" ]]; then
    FAILURE_MSG="Unknown failure."
  fi
  echo "$0: ${ERROR_RED}$FAILURE_MSG${NORMAL}"
  exit 1
}

# Wraps JavaScript code in a way that can be embedded in a worker.
# To get the inner source code, include the file generated by this function,
# then use name.toString() where `name` is the name passed into this
# function.
wrap-worker-code ( ) {
  name="$1"
  js="$2"
  echo "// Autogenerated code. Do not modify!"
  printf "function %s () {\n" "${name}"
  cat "${js}"
  printf "\n}\n"
}

################################ Main script ################################

run_tests=0
fetch_deps=1
unit_tests_only=0

# Process command-line arguments
while [[ $# -gt 0 ]] ; do
  key="$1"
  case $key in
    -clean)
      clean
      ;;
    -help|-h)
      display_usage
      exit
      ;;
    -test)
      run_tests=1
      ;;
{'fraction_non_alphanumeric': 0.13468501086169443, 'fraction_numerical': 0.005792903692976104, 'mean_word_length': 3.0387426900584797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2990304', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1725, 'n_words': 626}