<commit_before>#!/bin/bash
#
# Installer for DMI-TCAT on Linux.
#
# This script prompts the user interactively for parameters, or it can
# be run in batch mode (with the -b option).
#
# The paramaters can also be loaded from a configuration file (with
# the -c option).  The format of the config file is a Bash shell file
# that sets values for some/all of the parameter environment variables
# from the top section of this file (since the config file is simply
# sourced by this script).
#
# To be able to run in batch mode, a config file *must* minimally
# provide values for CONSUMERKEY, CONSUMERSECRET, USERTOKEN and
# USERSECRET.
#
# Run with -h for help.
#
# Supported distributions:
# - Ubuntu 18.04
# - Debian 9.*
#
#----------------------------------------------------------------

# TCAT Installer parameters

# Convention for Boolean: 'y' true; all other values (e.g. blank or 'n') false

# Twitter API credentials and capture options

CONSUMERKEY=
CONSUMERSECRET=
USERTOKEN=
USERSECRET=

CAPTURE_MODE=1 # 1=track phrases/keywords, 2=follow users, 3=onepercent

URLEXPANDYES=y # install URL Expander or not

# Apache

SERVERNAME= # should default to this machine's IP address (-s overrides)

# TCAT
TCAT_AUTO_UPDATE=0 # 0=off, 1=trivial, 2=substantial, 3=expensive

# Unix user and group to own the TCAT files

SHELLUSER=tcat
SHELLGROUP=tcat

# MySQL (blank password means randomly generate it)

DBPASS= # password for the MySQL "root" administrator account

TCATMYSQLUSER=tcatdbuser
TCATMYSQLPASS=

DB_CONFIG_MEMORY_PROFILE=y

LETSENCRYPT=n

# TCAT Web user interface logins (blank password means randomly generate it)

TCATADMINUSER=admin
TCATADMINPASS=

TCATUSER=tcat
TCATPASS=

# End of TCAT installer parameters

#----------------------------------------------------------------
# Error checking

PROG=`basename "$0"`

# Trap to abort script when a command fails.

trap "echo $PROG: command failed: install aborted; exit 3" ERR
# Can't figure out which command failed? Run using "bash -x" or uncomment:
# set -x # write each command to stderr before it is exceuted

# set -e # fail if a command fails (this works in sh, since trap ERR does not)

set -u # fail on attempts to expand undefined variables

#----------------------------------------------------------------
# Other constants (these should not need changing)

# Release of DMI-TCAT to install
# These can be changed using the -R and -B command line options

TCAT_GIT_REPOSITORY=https://github.com/digitalmethodsinitiative/dmi-tcat.git
TCAT_GIT_BRANCH= # empty string means use shallow clone of 'master' branch
                 # non-empty string means use a full clone of the named branch

# Where the MySQL defaults files are written

MYSQL_CNF_PREFIX='/etc/mysql/conf.d/tcat-'
MYSQL_CNF_SUFFIX='.cnf'

# Where the TCAT logins are written

TCAT_CNF_PREFIX='/etc/apache2/tcat-login-'
TCAT_CNF_SUFFIX='.txt'

# Where the TCAT logins for Apache Basic Authentication credentials are written

APACHE_PASSWORDS_FILE=/etc/apache2/tcat.htpasswd

# Where to install TCAT files

TCAT_DIR=/var/www/dmi-tcat

# Apache user and group

WEBUSER=www-data
WEBGROUP=www-data

#----------------------------------------------------------------
# Functions

# Prompts for a non-blank string
#
# Usage: promptStr promptMessage defaultValue
#
# Outputs: the string value

promptStr() {
    P_PROMPT=$1
    if [ $# -gt 1 ]; then
	P_DEFAULT=$2
    else
	P_DEFAULT=
    fi

    if [ -n "$P_DEFAULT" ]; then
	P_PROMPT="$P_PROMPT [$P_DEFAULT]"
    fi

    P_INPUT=
    while [ -z "$P_INPUT" ]; do
	read -p "$P_PROMPT: " P_INPUT
	if [ -z "$P_INPUT" ] ; then
	    P_INPUT="$P_DEFAULT"
	fi
    done
    echo $P_INPUT

    unset P_PROMPT
    unset P_DEFAULT
}

# Prompts for "y" or "n".
#
# Usage: promptYN promptMessage [defaultValue]
#
# Outputs: nothing
#
# Returns: 0 if "y", 1 if "n", or exits script if "q"

promptYN() {
    if [ $# -gt 1 ]; then
	P_DEFAULT="$2"
    else
	P_DEFAULT=
    fi

    if [ "$P_DEFAULT" = 'y' ]; then
	P_PROMPT="$1 [Y/n]? "
    elif [ "$P_DEFAULT" = 'n' ]; then
	P_PROMPT="$1 [y/N]? "
    else
	P_PROMPT="$1 [y/n]? "
    fi

    P_INPUT=
    while [ "$P_INPUT" != 'y' -a "$P_INPUT" != 'n' ]; do
	read -p "$P_PROMPT" P_INPUT
	if [ "$P_INPUT" = 'q' -o "$P_INPUT" = 'quit' -o \
	    "$P_INPUT" = 'Q' -o "$P_INPUT" = 'QUIT' ]; then
	    echo "$PROG: aborted by user" >&2
	    exit 1
	elif [ "$P_INPUT" = 'yes' -o \
	    "$P_INPUT" = 'Y' -o "$P_INPUT" = "YES" ]; then
	    P_INPUT='y'
	elif [ "$P_INPUT" = 'no' -o \
	    "$P_INPUT" = 'N' -o "$P_INPUT" = "NO" ]; then
	    P_INPUT='n'
	elif [ -z "$P_INPUT" ]; then
	    P_INPUT="$P_DEFAULT"
	fi
    done

    unset P_PROMPT
    unset P_DEFAULT

    if [ "$P_INPUT" = 'y' ]; then
	unset P_INPUT
	return 0
    else
	unset P_INPUT
	return 1
    fi
}

# Prompts for a password
#
# Usage: promptPassword promptMessage
#
# Usually call it like this:
#   VALUE=$(promptPassword "message"); echo
#
# Outputs: nothing
#
# Returns: the password, or the empty string.

promptPassword() {
    P_PROMPT="$1 [default: randomly generated]: "
    P_INPUT=""
    while [ -z "$P_INPUT" ]; do
	read -s -p "$P_PROMPT" P_INPUT
	if [ -z "$P_INPUT" ]; then
	    # Blank password means to randomly generate it
	    break
	else
	    # Confirm password
	    echo >&2
	    read -s -p "Please reenter password: " P2
	    if [ "$P_INPUT" != "$P2" ]; then
		echo >&2
		echo "Error: passwords did not match, please try again." >&2
		P_INPUT="" # clear to re-prompting
	    fi
	fi
    done

    echo "$P_INPUT" # the password or the empty string

    unset P_PROMPT
    unset P_INPUT
}

#----------------------------------------------------------------
# Process command line

SHORT_OPTS="B:bc:fGhlR:s:U"
if ! getopt $SHORT_OPTS "$@" >/dev/null; then
    echo "$PROG: usage error (use -h for help)" >&2
    exit 2
fi
ARGS=`getopt $SHORT_OPTS "$@"`
eval set -- $ARGS

## Process parsed options

BATCH_MODE=
CONFIG_FILE=
GEO_SEARCH=y
CMD_SERVERNAME=
DO_UPDATE_UPGRADE=y
DO_SAVE_TCAT_LOGINS=
FORCE_INSTALL=
HELP=

while [ $# -gt 0 ]; do
    case "$1" in
        -B) TCAT_GIT_BRANCH="$2"; shift;;
        -b) BATCH_MODE=y;;
	-c) CONFIG_FILE="$2"; shift;;
	-f) FORCE_INSTALL=y;;
	-G) GEO_SEARCH=n;;
	-l) DO_SAVE_TCAT_LOGINS=y;;
        -R) TCAT_GIT_REPOSITORY="$2"; shift;;
        -s) CMD_SERVERNAME="$2"; shift;;
	-U) DO_UPDATE_UPGRADE=n;;
        -h) HELP='y';;
	--) break;;
    esac
    shift
done

if [ -n "$HELP" ]; then
    cat <<EOF
Usage: $PROG [options]
Options:
  -b             run in batch mode
  -c configFile  load parameters from file
  -s server      the name or IP address of this machine

  -l             save a copy of TCAT login username and passwords in plain text

  -G             install without geographical search (for Ubuntu < 15.x)
  -U             do not run apt-get update and apt-get upgrade

  -h             show this help message

  -R repoURL     GIT repository for TCAT (default: official DMI-TCAT on GitHub)
  -B branch      GIT branch in repository to install (default: master)
  -f             force install attempt on unsupported distributions
                 (for developers only: install or TCAT will probably not work)
EOF
    exit 0
fi

if [ $# -gt 1 ]; then
    echo "$PROG: too many arguments (-h for help)" >&2
    exit 2
fi

#----------------------------------------------------------------
# Checks

# Script run with root privileges (either as root or with sudo)

if [ $(id -u) -ne 0 ]; then
   tput setaf 1
   echo "$PROG: error: this script was run without root privileges" 1>&2
   tput sgr0
   exit 1
fi

# Expected OS version

OS=`uname -s`
if [ $OS != 'Linux' ]; then
    echo "$PROG: error: unsupported operating system: not Linux: $OS" >&2
    exit 1
fi

if ! which lsb_release >/dev/null 2>&1; then
    echo "$PROG: error: unsupported distribution: missing lsb_release" >&2
    exit 1
fi

DISTRIBUTION_ID=`lsb_release -i -s`

if [ "$DISTRIBUTION_ID" = 'Ubuntu' ]; then
    UBUNTU_VERSION=`lsb_release -r -s`
    DEBIAN_VERSION=
    UBUNTU_VERSION_MAJOR=$(echo $UBUNTU_VERSION |
	awk -F . '{if (match($1, /^[0-9]+$/)) print $1}')

    if [ -z "$UBUNTU_VERSION_MAJOR" ]; then
	echo "$PROG: error: unexpected Ubuntu version: $UBUNTU_VERSION" >&2
	exit 1
    fi
    if [ "$UBUNTU_VERSION" != '18.04' ] || [ "$UBUNTU_VERSION" != '20.04' ]; then
	if [ -z "$FORCE_INSTALL" ]; then
	    echo "$PROG: error: unsupported distribution: Ubuntu $UBUNTU_VERSION" >&2
	    exit 1
	fi
    fi

    if [ "$UBUNTU_VERSION_MAJOR" -lt 15 ]; then
	echo "Warning: no geographical search on Ubuntu $UBUNTU_VERSION < 15.x"

	if [ "$GEO_SEARCH" = 'y' ]; then
	    if [ "$BATCH_MODE" = 'y' ]; then
		echo "$PROG: aborted (use -G to leave out geo search)" >&2
		exit 1
	    else
		if ! promptYN "Continue install without geographical search";
		then
		    echo "$PROG: aborted by user"<commit_msg>fix to add 20.04 version to tcat-install.linux.sh after testing<commit_after>#!/bin/bash
#
# Installer for DMI-TCAT on Linux.
#
# This script prompts the user interactively for parameters, or it can
# be run in batch mode (with the -b option).
#
# The paramaters can also be loaded from a configuration file (with
# the -c option).  The format of the config file is a Bash shell file
# that sets values for some/all of the parameter environment variables
# from the top section of this file (since the config file is simply
# sourced by this script).
#
# To be able to run in batch mode, a config file *must* minimally
# provide values for CONSUMERKEY, CONSUMERSECRET, USERTOKEN and
# USERSECRET.
#
# Run with -h for help.
#
# Supported distributions:
# - Ubuntu 18.04
# - Debian 9.*
#
#----------------------------------------------------------------

# TCAT Installer parameters

# Convention for Boolean: 'y' true; all other values (e.g. blank or 'n') false

# Twitter API credentials and capture options

CONSUMERKEY=
CONSUMERSECRET=
USERTOKEN=
USERSECRET=

CAPTURE_MODE=1 # 1=track phrases/keywords, 2=follow users, 3=onepercent

URLEXPANDYES=y # install URL Expander or not

# Apache

SERVERNAME= # should default to this machine's IP address (-s overrides)

# TCAT
TCAT_AUTO_UPDATE=0 # 0=off, 1=trivial, 2=substantial, 3=expensive

# Unix user and group to own the TCAT files

SHELLUSER=tcat
SHELLGROUP=tcat

# MySQL (blank password means randomly generate it)

DBPASS= # password for the MySQL "root" administrator account

TCATMYSQLUSER=tcatdbuser
TCATMYSQLPASS=

DB_CONFIG_MEMORY_PROFILE=y

LETSENCRYPT=n

# TCAT Web user interface logins (blank password means randomly generate it)

TCATADMINUSER=admin
TCATADMINPASS=

TCATUSER=tcat
TCATPASS=

# End of TCAT installer parameters

#----------------------------------------------------------------
# Error checking

PROG=`basename "$0"`

# Trap to abort script when a command fails.

trap "echo $PROG: command failed: install aborted; exit 3" ERR
# Can't figure out which command failed? Run using "bash -x" or uncomment:
# set -x # write each command to stderr before it is exceuted

# set -e # fail if a command fails (this works in sh, since trap ERR does not)

set -u # fail on attempts to expand undefined variables

#----------------------------------------------------------------
# Other constants (these should not need changing)

# Release of DMI-TCAT to install
# These can be changed using the -R and -B command line options

TCAT_GIT_REPOSITORY=https://github.com/digitalmethodsinitiative/dmi-tcat.git
TCAT_GIT_BRANCH= # empty string means use shallow clone of 'master' branch
                 # non-empty string means use a full clone of the named branch

# Where the MySQL defaults files are written

MYSQL_CNF_PREFIX='/etc/mysql/conf.d/tcat-'
MYSQL_CNF_SUFFIX='.cnf'

# Where the TCAT logins are written

TCAT_CNF_PREFIX='/etc/apache2/tcat-login-'
TCAT_CNF_SUFFIX='.txt'

# Where the TCAT logins for Apache Basic Authentication credentials are written

APACHE_PASSWORDS_FILE=/etc/apache2/tcat.htpasswd

# Where to install TCAT files

TCAT_DIR=/var/www/dmi-tcat

# Apache user and group

WEBUSER=www-data
WEBGROUP=www-data

#----------------------------------------------------------------
# Functions

# Prompts for a non-blank string
#
# Usage: promptStr promptMessage defaultValue
#
# Outputs: the string value

promptStr() {
    P_PROMPT=$1
    if [ $# -gt 1 ]; then
	P_DEFAULT=$2
    else
	P_DEFAULT=
    fi

    if [ -n "$P_DEFAULT" ]; then
	P_PROMPT="$P_PROMPT [$P_DEFAULT]"
    fi

    P_INPUT=
    while [ -z "$P_INPUT" ]; do
	read -p "$P_PROMPT: " P_INPUT
	if [ -z "$P_INPUT" ] ; then
	    P_INPUT="$P_DEFAULT"
	fi
    done
    echo $P_INPUT

    unset P_PROMPT
    unset P_DEFAULT
}

# Prompts for "y" or "n".
#
# Usage: promptYN promptMessage [defaultValue]
#
# Outputs: nothing
#
# Returns: 0 if "y", 1 if "n", or exits script if "q"

promptYN() {
    if [ $# -gt 1 ]; then
	P_DEFAULT="$2"
    else
	P_DEFAULT=
    fi

    if [ "$P_DEFAULT" = 'y' ]; then
	P_PROMPT="$1 [Y/n]? "
    elif [ "$P_DEFAULT" = 'n' ]; then
	P_PROMPT="$1 [y/N]? "
    else
	P_PROMPT="$1 [y/n]? "
    fi

    P_INPUT=
    while [ "$P_INPUT" != 'y' -a "$P_INPUT" != 'n' ]; do
	read -p "$P_PROMPT" P_INPUT
	if [ "$P_INPUT" = 'q' -o "$P_INPUT" = 'quit' -o \
	    "$P_INPUT" = 'Q' -o "$P_INPUT" = 'QUIT' ]; then
	    echo "$PROG: aborted by user" >&2
	    exit 1
	elif [ "$P_INPUT" = 'yes' -o \
	    "$P_INPUT" = 'Y' -o "$P_INPUT" = "YES" ]; then
	    P_INPUT='y'
	elif [ "$P_INPUT" = 'no' -o \
	    "$P_INPUT" = 'N' -o "$P_INPUT" = "NO" ]; then
	    P_INPUT='n'
	elif [ -z "$P_INPUT" ]; then
	    P_INPUT="$P_DEFAULT"
	fi
    done

    unset P_PROMPT
    unset P_DEFAULT

    if [ "$P_INPUT" = 'y' ]; then
	unset P_INPUT
	return 0
    else
	unset P_INPUT
	return 1
    fi
}

# Prompts for a password
#
# Usage: promptPassword promptMessage
#
# Usually call it like this:
#   VALUE=$(promptPassword "message"); echo
#
# Outputs: nothing
#
# Returns: the password, or the empty string.

promptPassword() {
    P_PROMPT="$1 [default: randomly generated]: "
    P_INPUT=""
    while [ -z "$P_INPUT" ]; do
	read -s -p "$P_PROMPT" P_INPUT
	if [ -z "$P_INPUT" ]; then
	    # Blank password means to randomly generate it
	    break
	else
	    # Confirm password
	    echo >&2
	    read -s -p "Please reenter password: " P2
	    if [ "$P_INPUT" != "$P2" ]; then
		echo >&2
		echo "Error: passwords did not match, please try again." >&2
		P_INPUT="" # clear to re-prompting
	    fi
	fi
    done

    echo "$P_INPUT" # the password or the empty string

    unset P_PROMPT
    unset P_INPUT
}

#----------------------------------------------------------------
# Process command line

SHORT_OPTS="B:bc:fGhlR:s:U"
if ! getopt $SHORT_OPTS "$@" >/dev/null; then
    echo "$PROG: usage error (use -h for help)" >&2
    exit 2
fi
ARGS=`getopt $SHORT_OPTS "$@"`
eval set -- $ARGS

## Process parsed options

BATCH_MODE=
CONFIG_FILE=
GEO_SEARCH=y
CMD_SERVERNAME=
DO_UPDATE_UPGRADE=y
DO_SAVE_TCAT_LOGINS=
FORCE_INSTALL=
HELP=

while [ $# -gt 0 ]; do
    case "$1" in
        -B) TCAT_GIT_BRANCH="$2"; shift;;
        -b) BATCH_MODE=y;;
	-c) CONFIG_FILE="$2"; shift;;
	-f) FORCE_INSTALL=y;;
	-G) GEO_SEARCH=n;;
	-l) DO_SAVE_TCAT_LOGINS=y;;
        -R) TCAT_GIT_REPOSITORY="$2"; shift;;
        -s) CMD_SERVERNAME="$2"; shift;;
	-U) DO_UPDATE_UPGRADE=n;;
        -h) HELP='y';;
	--) break;;
    esac
    shift
done

if [ -n "$HELP" ]; then
    cat <<EOF
Usage: $PROG [options]
Options:
  -b             run in batch mode
  -c configFile  load parameters from file
  -s server      the name or IP address of this machine

  -l             save a copy of TCAT login username and passwords in plain text

  -G             install without geographical search (for Ubuntu < 15.x)
  -U             do not run apt-get update and apt-get upgrade

  -h             show this help message

  -R repoURL     GIT repository for TCAT (default: official DMI-TCAT on GitHub)
  -B branch      GIT branch in repository to install (default: master)
  -f             force install attempt on unsupported distributions
                 (for developers only: install or TCAT will probably not work)
EOF
    exit 0
fi

if [ $# -gt 1 ]; then
    echo "$PROG: too many arguments (-h for help)" >&2
    exit 2
fi

#----------------------------------------------------------------
# Checks

# Script run with root privileges (either as root or with sudo)

if [ $(id -u) -ne 0 ]; then
   tput setaf 1
   echo "$PROG: error: this script was run without root privileges" 1>&2
   tput sgr0
   exit 1
fi

# Expected OS version

OS=`uname -s`
if [ $OS != 'Linux' ]; then
    echo "$PROG: error: unsupported operating system: not Linux: $OS" >&2
    exit 1
fi

if ! which lsb_release >/dev/null 2>&1; then
    echo "$PROG: error: unsupported distribution: missing lsb_release" >&2
    exit 1
fi

DISTRIBUTION_ID=`lsb_release -i -s`

if [ "$DISTRIBUTION_ID" = 'Ubuntu' ]; then
    UBUNTU_VERSION=`lsb_release -r -s`
    DEBIAN_VERSION=
    UBUNTU_VERSION_MAJOR=$(echo $UBUNTU_VERSION |
	awk -F . '{if (match($1, /^[0-9]+$/)) print $1}')

    if [ -z "$UBUNTU_VERSION_MAJOR" ]; then
	echo "$PROG: error: unexpected Ubuntu version: $UBUNTU_VERSION" >&2
	exit 1
    fi
    if [ "$UBUNTU_VERSION" != '18.04' ] && [ "$UBUNTU_VERSION" != '20.04' ]; then
	if [ -z "$FORCE_INSTALL" ]; then
	    echo "$PROG: error: unsupported distribution: Ubuntu $UBUNTU_VERSION" >&2
	    exit 1
	fi
    fi

    if [ "$UBUNTU_VERSION_MAJOR" -lt 15 ]; then
	echo "Warning: no geographical search on Ubuntu $UBUNTU_VERSION < 15.x"

	if [ "$GEO_SEARCH" = 'y' ]; then
	    if [ "$BATCH_MODE" = 'y' ]; then
		echo "$PROG: aborted (use -G to leave out geo search)" >&2
		exit 1
	    else
		if ! promptYN "Continue install without geographical search";
		then
		    echo "$PROG: aborted by user"
{'fraction_non_alphanumeric': 0.1563171586924556, 'fraction_numerical': 0.009953625155525394, 'mean_word_length': 2.64673128480099, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3490690', 'n_tokens_mistral': 6507, 'n_tokens_neox': 6099, 'n_words': 2065}