<commit_before>#! /bin/sh
#
# Load a TSV file into a table using PostgreSQL COPY FROM command
#
# First argument is the database name
# Second argument is the table name
# Third argument is the path to the TSV file or to a directory containing tsv
# files

abs_real_path () { 
	case "$1" in 
		/*) TSV_FILE_ABS_PATH=`readlink -f $1`
			;;
		*)  TSV_FILE_ABS_PATH=`readlink -f $PWD/$1`
			;;
	esac; 
}

copy_from_file() {
	SQL_COMMAND_FILE=`mktemp /tmp/ctff.XXXXX` || exit 1
	abs_real_path $1
	echo "COPY $TABLE FROM '${TSV_FILE_ABS_PATH}';" > ${SQL_COMMAND_FILE}
	psql -U $DBUSER -h $DBHOST -p $DBPORT -X --set ON_ERROR_STOP=1 -d $DB -f ${SQL_COMMAND_FILE}
	rm ${SQL_COMMAND_FILE}
}

if [ $# -ne 3 ]; then
	echo "$0: ERROR: wrong number of arguments" >&2
	echo "$0: USAGE: $0 DB TABLE FILE/DIR" >&2
	exit 1
fi

if [ ! -r $3 ]; then
	echo "$0: ERROR: TSV file/directory not readable" >&2
	exit 1
fi

DB=$1
TABLE=$2
<commit_msg>Check multiple environment variables in copy_table_from_file.sh<commit_after>#!/bin/bash -e
#
# Load a TSV file into a table using PostgreSQL COPY FROM command
#
# First argument is the database name
# Second argument is the table name
# Third argument is the path to the TSV file or to a directory containing tsv
# files

abs_real_path () { 
	case "$1" in 
		/*) TSV_FILE_ABS_PATH=`readlink -f $1`
			;;
		*)  TSV_FILE_ABS_PATH=`readlink -f $PWD/$1`
			;;
	esac; 
}

copy_from_file() {
	SQL_COMMAND_FILE=`mktemp /tmp/ctff.XXXXX` || exit 1
	abs_real_path $1
	echo "COPY $TABLE FROM '${TSV_FILE_ABS_PATH}';" > ${SQL_COMMAND_FILE}
	psql -U ${DD_DBUSER-${PGUSER-${DBUSER-$USER}}} -h ${DD_DBHOST-${PGHOST-$DBHOST}} -p ${DD_DBPORT-${PGPORT-${DBPORT-5432}}} -X --set ON_ERROR_STOP=1 -d $DB -f ${SQL_COMMAND_FILE}
	rm ${SQL_COMMAND_FILE}
}

if [ $# -ne 3 ]; then
	echo "$0: ERROR: wrong number of arguments" >&2
	echo "$0: USAGE: $0 DB TABLE FILE/DIR" >&2
	exit 1
fi

if [ ! -r $3 ]; then
	echo "$0: ERROR: TSV file/directory not readable" >&2
	exit 1
fi

DB=$1
TABLE=$2

{'fraction_non_alphanumeric': 0.14522613065326634, 'fraction_numerical': 0.021105527638190954, 'mean_word_length': 3.8325242718446604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3717930', 'n_tokens_mistral': 836, 'n_tokens_neox': 808, 'n_words': 249}