<commit_before>#!/usr/bin/env bash

#
# Common constants and functions used by the blackbox_* utilities.
#

# Usage:
#
#   set -e
#   source "${0%/*}/_blackbox_common.sh"

# Load additional useful functions
source "${0%/*}"/_stack_lib.sh

# Where are we?
: "${BLACKBOX_HOME:="$(cd "${0%/*}" ; pwd)"}" ;

# What are the candidates for the blackbox data directory?
declare -a BLACKBOXDATA_CANDIDATES
BLACKBOXDATA_CANDIDATES=(
  'keyrings/live'
  '.blackbox'
)

# If $EDITOR is not set, set it to "vi":
: "${EDITOR:=vi}" ;

# Allow overriding gpg command
: "${GPG:=gpg}" ;

function physical_directory_of() {
  local d=$(dirname "$1")
  local f=$(basename "$1")
  (cd "$d" && echo "$(pwd -P | sed 's/\/$//')/$f" )
}

# Set REPOBASE to the top of the repository
# Set VCS_TYPE to 'git', 'hg', 'svn' or 'unknown'
if which >/dev/null 2>/dev/null git && git rev-parse --show-toplevel >/dev/null 2>&1 ; then
  VCS_TYPE=git
  REPOBASE=$(git rev-parse --show-toplevel)
elif [ -d ".svn" ] ; then
  # Find topmost dir with .svn sub-dir
  parent=""
  grandparent="."
  while [ -d "$grandparent/.svn" ]; do
    parent=$grandparent
    grandparent="$parent/.."
  done

  REPOBASE=$(cd "$parent" ; pwd)
  VCS_TYPE=svn
elif which >/dev/null 2>/dev/null hg && hg root >/dev/null 2>&1 ; then
  # NOTE: hg has to be tested last because it always "succeeds".
  VCS_TYPE=hg
  REPOBASE=$(hg root 2>/dev/null)
else
  # We aren't in a repo at all.  Assume the cwd is the root
  # of the tree.
  VCS_TYPE=unknown
  REPOBASE="$(pwd)"
fi
export VCS_TYPE
export REPOBASE=$(physical_directory_of "$REPOBASE")
# FIXME: Verify this function by checking for .hg or .git
# after determining what we believe to be the answer.

if [[ -n "$BLACKBOX_REPOBASE" ]]; then
	echo "Using custom repobase: $BLACKBOX_REPOBASE" >&2
	export REPOBASE="$BLACKBOX_REPOBASE"
fi

if [ -z "$BLACKBOXDATA" ] ; then
  BLACKBOXDATA="${BLACKBOXDATA_CANDIDATES[0]}"
  for candidate in ${BLACKBOXDATA_CANDIDATES[@]} ; do
    if [ -d "$REPOBASE/$candidate" ] ; then
      BLACKBOXDATA="$candidate"
      break
    fi
  done
fi

KEYRINGDIR="$REPOBASE/$BLACKBOXDATA"
BB_ADMINS_FILE="blackbox-admins.txt"
BB_ADMINS="${KEYRINGDIR}/${BB_ADMINS_FILE}"
BB_FILES_FILE="blackbox-files.txt"
BB_FILES="${KEYRINGDIR}/${BB_FILES_FILE}"
SECRING="${KEYRINGDIR}/secring.gpg"
: "${DECRYPT_UMASK:=0022}" ;
# : ${DECRYPT_UMASK:=o=} ;

# Checks if $1 is 0 bytes, and if $1/keyrings
# is a directory
function is_blackbox_repo() {
  if [[ -n "$1" ]] && [[ -d "$1/keyrings" ]]; then
    return 0 # Yep, its a repo
  else
    return 1
  fi
}

# Return error if not on cryptlist.
function is_on_cryptlist() {
  # Assumes $1 does NOT have the .gpg extension
  file_contains_line "$BB_FILES" "$(vcs_relative_path "$1")"
}

# Exit with error if a file exists.
function fail_if_exists() {
  if [[ -f "$1" ]]; then
    echo ERROR: "$1" exists.  "$2" >&2
    echo Exiting... >&2
    exit 1
  fi
}

# Exit with error if a file is missing.
function fail_if_not_exists() {
  if [[ ! -f "$1" ]]; then
    echo ERROR: "$1" not found.  "$2" >&2
    echo Exiting... >&2
    exit 1
  fi
}

# Exit we we aren't in a VCS repo.
function fail_if_not_in_repo() {
  if [[ $VCS_TYPE = "unknown" ]]; then
    echo "ERROR: This must be run in a VCS repo: git, hg, or svn." >&2
    echo Exiting... >&2
    exit 1
  fi
}

# Exit with error if filename is not registered on blackbox list.
function fail_if_not_on_cryptlist() {
  # Assumes $1 does NOT have the .gpg extension

  local name="$1"

  if ! is_on_cryptlist "$name" ; then
    echo "ERROR: $name not found in $BB_FILES" >&2
    echo "PWD=$(/bin/pwd)" >&2
    echo 'Exiting...' >&2
    exit 1
  fi
}

# Exit with error if keychain contains secret keys.
function fail_if_keychain_has_secrets() {
  if [[ -s ${SECRING} ]]; then
    echo 'ERROR: The file' "$SECRING" 'should be empty.' >&2
    echo 'Did someone accidentally add this private key to the ring?' >&2
    echo 'Exiting...' >&2
    exit 1
  fi
}

function get_pubring_path() {
  if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then
    echo "${KEYRINGDIR}/pubring.gpg"
  else
    echo "${KEYRINGDIR}/pubring.kbx"
  fi
}

# Output the unencrypted filename.
function get_unencrypted_filename() {
  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
}

# Output the encrypted filename.
function get_encrypted_filename() {
  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
}

# Prepare keychain for use.
function prepare_keychain() {
  local keyringasc
  echo '========== Importing keychain: START' >&2
  # Works with gpg 2.0
  #$GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
  # Works with gpg 2.0 and 2.1
  # NB: We must export the keys to a format that can be imported.
  make_self_deleting_tempfile keyringasc
  export LANG="C.UTF-8"
  $GPG --export --keyring "$(get_pubring_path)" >"$keyringasc"
  $GPG --import "$keyringasc" 2>&1 | egrep -v 'not changed$' >&2
  echo '========== Importing keychain: DONE' >&2
}

# Add file to list of encrypted files.
function add_filename_to_cryptlist() {
  # If the name is already on the list, this is a no-op.
  # However no matter what the datestamp is updated.
  
  # https://github.com/koalaman/shellcheck/wiki/SC2155
  local name
  name=$(vcs_relative_path "$1")

  if file_contains_line "$BB_FILES" "$name" ; then
    echo "========== File is registered. No need to add to list."
  else
    echo "========== Adding file to list."
    touch "$BB_FILES"
    sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES"
  fi
}

# Removes a file from the list of encrypted files<commit_msg>Exclude our default keyring from import<commit_after>#!/usr/bin/env bash

#
# Common constants and functions used by the blackbox_* utilities.
#

# Usage:
#
#   set -e
#   source "${0%/*}/_blackbox_common.sh"

# Load additional useful functions
source "${0%/*}"/_stack_lib.sh

# Where are we?
: "${BLACKBOX_HOME:="$(cd "${0%/*}" ; pwd)"}" ;

# What are the candidates for the blackbox data directory?
declare -a BLACKBOXDATA_CANDIDATES
BLACKBOXDATA_CANDIDATES=(
  'keyrings/live'
  '.blackbox'
)

# If $EDITOR is not set, set it to "vi":
: "${EDITOR:=vi}" ;

# Allow overriding gpg command
: "${GPG:=gpg}" ;

function physical_directory_of() {
  local d=$(dirname "$1")
  local f=$(basename "$1")
  (cd "$d" && echo "$(pwd -P | sed 's/\/$//')/$f" )
}

# Set REPOBASE to the top of the repository
# Set VCS_TYPE to 'git', 'hg', 'svn' or 'unknown'
if which >/dev/null 2>/dev/null git && git rev-parse --show-toplevel >/dev/null 2>&1 ; then
  VCS_TYPE=git
  REPOBASE=$(git rev-parse --show-toplevel)
elif [ -d ".svn" ] ; then
  # Find topmost dir with .svn sub-dir
  parent=""
  grandparent="."
  while [ -d "$grandparent/.svn" ]; do
    parent=$grandparent
    grandparent="$parent/.."
  done

  REPOBASE=$(cd "$parent" ; pwd)
  VCS_TYPE=svn
elif which >/dev/null 2>/dev/null hg && hg root >/dev/null 2>&1 ; then
  # NOTE: hg has to be tested last because it always "succeeds".
  VCS_TYPE=hg
  REPOBASE=$(hg root 2>/dev/null)
else
  # We aren't in a repo at all.  Assume the cwd is the root
  # of the tree.
  VCS_TYPE=unknown
  REPOBASE="$(pwd)"
fi
export VCS_TYPE
export REPOBASE=$(physical_directory_of "$REPOBASE")
# FIXME: Verify this function by checking for .hg or .git
# after determining what we believe to be the answer.

if [[ -n "$BLACKBOX_REPOBASE" ]]; then
	echo "Using custom repobase: $BLACKBOX_REPOBASE" >&2
	export REPOBASE="$BLACKBOX_REPOBASE"
fi

if [ -z "$BLACKBOXDATA" ] ; then
  BLACKBOXDATA="${BLACKBOXDATA_CANDIDATES[0]}"
  for candidate in ${BLACKBOXDATA_CANDIDATES[@]} ; do
    if [ -d "$REPOBASE/$candidate" ] ; then
      BLACKBOXDATA="$candidate"
      break
    fi
  done
fi

KEYRINGDIR="$REPOBASE/$BLACKBOXDATA"
BB_ADMINS_FILE="blackbox-admins.txt"
BB_ADMINS="${KEYRINGDIR}/${BB_ADMINS_FILE}"
BB_FILES_FILE="blackbox-files.txt"
BB_FILES="${KEYRINGDIR}/${BB_FILES_FILE}"
SECRING="${KEYRINGDIR}/secring.gpg"
: "${DECRYPT_UMASK:=0022}" ;
# : ${DECRYPT_UMASK:=o=} ;

# Checks if $1 is 0 bytes, and if $1/keyrings
# is a directory
function is_blackbox_repo() {
  if [[ -n "$1" ]] && [[ -d "$1/keyrings" ]]; then
    return 0 # Yep, its a repo
  else
    return 1
  fi
}

# Return error if not on cryptlist.
function is_on_cryptlist() {
  # Assumes $1 does NOT have the .gpg extension
  file_contains_line "$BB_FILES" "$(vcs_relative_path "$1")"
}

# Exit with error if a file exists.
function fail_if_exists() {
  if [[ -f "$1" ]]; then
    echo ERROR: "$1" exists.  "$2" >&2
    echo Exiting... >&2
    exit 1
  fi
}

# Exit with error if a file is missing.
function fail_if_not_exists() {
  if [[ ! -f "$1" ]]; then
    echo ERROR: "$1" not found.  "$2" >&2
    echo Exiting... >&2
    exit 1
  fi
}

# Exit we we aren't in a VCS repo.
function fail_if_not_in_repo() {
  if [[ $VCS_TYPE = "unknown" ]]; then
    echo "ERROR: This must be run in a VCS repo: git, hg, or svn." >&2
    echo Exiting... >&2
    exit 1
  fi
}

# Exit with error if filename is not registered on blackbox list.
function fail_if_not_on_cryptlist() {
  # Assumes $1 does NOT have the .gpg extension

  local name="$1"

  if ! is_on_cryptlist "$name" ; then
    echo "ERROR: $name not found in $BB_FILES" >&2
    echo "PWD=$(/bin/pwd)" >&2
    echo 'Exiting...' >&2
    exit 1
  fi
}

# Exit with error if keychain contains secret keys.
function fail_if_keychain_has_secrets() {
  if [[ -s ${SECRING} ]]; then
    echo 'ERROR: The file' "$SECRING" 'should be empty.' >&2
    echo 'Did someone accidentally add this private key to the ring?' >&2
    echo 'Exiting...' >&2
    exit 1
  fi
}

function get_pubring_path() {
  if [[ -f "${KEYRINGDIR}/pubring.gpg" ]]; then
    echo "${KEYRINGDIR}/pubring.gpg"
  else
    echo "${KEYRINGDIR}/pubring.kbx"
  fi
}

# Output the unencrypted filename.
function get_unencrypted_filename() {
  echo "$(dirname "$1")/$(basename "$1" .gpg)" | sed -e 's#^\./##'
}

# Output the encrypted filename.
function get_encrypted_filename() {
  echo "$(dirname "$1")/$(basename "$1" .gpg).gpg" | sed -e 's#^\./##'
}

# Prepare keychain for use.
function prepare_keychain() {
  local keyringasc
  echo '========== Importing keychain: START' >&2
  # Works with gpg 2.0
  #$GPG --import "$(get_pubring_path)" 2>&1 | egrep -v 'not changed$' >&2
  # Works with gpg 2.0 and 2.1
  # NB: We must export the keys to a format that can be imported.
  make_self_deleting_tempfile keyringasc
  export LANG="C.UTF-8"
  $GPG --export --no-default-keyring --keyring "$(get_pubring_path)" >"$keyringasc"
  $GPG --import "$keyringasc" 2>&1 | egrep -v 'not changed$' >&2
  echo '========== Importing keychain: DONE' >&2
}

# Add file to list of encrypted files.
function add_filename_to_cryptlist() {
  # If the name is already on the list, this is a no-op.
  # However no matter what the datestamp is updated.
  
  # https://github.com/koalaman/shellcheck/wiki/SC2155
  local name
  name=$(vcs_relative_path "$1")

  if file_contains_line "$BB_FILES" "$name" ; then
    echo "========== File is registered. No need to add to list."
  else
    echo "========== Adding file to list."
    touch "$BB_FILES"
    sort -u -o "$BB_FILES" <(echo "$name") "$BB_FILES"
  fi
}

# Removes a file from the list of encrypted files
{'fraction_non_alphanumeric': 0.15960010711416586, 'fraction_numerical': 0.01356779434080157, 'mean_word_length': 3.519564340459863, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 61, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5250428', 'n_tokens_mistral': 4339, 'n_tokens_neox': 4164, 'n_words': 1285}