<commit_before>
# Source this for CDM shell functions

log "Reading cdm_lib.sh ..."

# Do a CDM lookup
cdm_lookup() {
	CDM_PL=$1
	CDM_FILE=$2
	FILE=$3

	RESULT="DEFAULT"
	if [ ! -z $CDM_FILE ] && [ -f $CDM_PL ] && [ -f $CDM_FILE ]; then
		RESULT=$( perl $CDM_PL lookup $FILE < $CDM_FILE 2> $INFO )
		checkError 254 "cdm_lookup(): failed! (malformed CDM file?)"
	fi
	echo $RESULT
}

cdm_action() {
	log "CDM_ACTION: $@"

	local JOBDIR=$1  # Given jobdir
	local MODE=$2    # INPUT or OUTPUT
	local FILE=$3    # User file
	local POLICY=$4  # DIRECT, BROADCAST, ...
	shift 4
	local ARGS=$@

	log "POLICY=$POLICY"
	case $POLICY in
		DIRECT)
			DIRECT_DIR=${ARGS[0]}
			log "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE via $JOBDIR/$FILE"
			if [ $MODE == "INPUT" ]; then
				[ -f "$DIRECT_DIR/$FILE" ]
				checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!"
				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
			elif [ $MODE == "OUTPUT" ]; then
				mkdir -p $DIRECT_DIR
				checkError 254 "CDM[DIRECT]: mkdir -p $DIRECT_DIR failed!"
				mkdir -p $( dirname $DIRECT_DIR/$FILE )
				checkError 254 "CDM[DIRECT]: mkdir -p $( dirname $FILE ) failed!"
				touch $DIRECT_DIR/$FILE
				checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!"
				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
			else
				fail 254 "Unknown MODE: $MODE"
			fi
			;;
 		LOCAL)
 			TOOL=${ARGS[0]}
 			REMOTE_DIR=${ARGS[1]}
 			FLAGS=${ARGS[3]}
 			log "CDM[LOCAL]: Copying $DIRECT_DIR/$FILE to $JOBDIR/$FILE"
 			if [ $MODE == "INPUT" ]; then
 				[ -f "$DIRECT_DIR/$FILE" ]
 				checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!"
 				$TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
 				checkError 254 "CDM[LOCAL]: Tool failed!"
 			elif [ $MODE == "OUTPUT" ]; then
 				log "CDM[LOCAL]..."
 			else
 				fail 254 "Unknown MODE: $MODE"
 			fi
 			;;
		BROADCAST)
			BROADCAST_DIR=${ARGS[0]}
			if [ $MODE == "INPUT" ]; then
				log "CDM[BROADCAST]: Linking $JOBDIR/$FILE to $BROADCAST_DIR/$FILE"
				[ -f "$BROADCAST_DIR/$FILE" ]
				checkError 254 "CDM[BROADCAST]: $BROADCAST_DIR/$FILE does not exist!"
				ln -s $BROADCAST_DIR/$FILE $JOBDIR/$FILE
				checkError 254 "CDM[BROADCAST]: Linking to $BROADCAST_DIR/$FILE failed!"
			else
				echo "CDM[BROADCAST]: Skipping output file: ${FILE}"
			fi
			;;
		GATHER)
			if [ $MODE == "INPUT" ]; then
				fail 254 "Cannot GATHER an input file!"
			fi
    esac
}

# Setup GATHER_DIR and some variables
cdm_gather_setup() {
	GATHER_DIR=$1

	logstate "GATHER_DIR $GATHER_DIR"
	mkdir -p $GATHER_DIR
	checkError 254 "Could not create: $GATHER_DIR"

	GATHER_LOCKFILE=$GATHER_DIR/.cdm.lock
	GATHER_MY_FILE=$GATHER_DIR/.cdm-$ID.lock
	GATHER_MY_OUTBOX=$GATHER_DIR/.cdm-outgoing-$ID
}

# Acquire the GATHER_LOCKFILE
cdm_gather_lock_acquire() {
	TRYING=1
	COUNT=0

	touch $GATHER_MY_FILE
	checkError 254 "Could not touch my file: $GATHER_MY_FILE"
	logstate "LOCK_ACQUIRE $GATHER_MY_FILE"
	while (( TRYING )); do
		ln $GATHER_MY_FILE $GATHER_LOCKFILE
		CODE=$?
		TRYING=$CODE
		if (( TRYING )); then
			logstate "LOCK_DELAY $GATHER_MY_FILE : $CODE"
			if (( COUNT++ > 10 )); then
				fail 254 "Could not acquire lock!"
				exit 1
			fi
			sleep $(( RANDOM % 10 ))
		fi
	done
}

# Move files from JOBDIR to GATHER_DIR
cdm_gather_import() {
	pushd jobs/$JOBDIR/$ID
	logstate "GATHER_IMPORT $GATHER_OUTPUT"
	mv $GATHER_OUTPUT $GATHER_DIR
	checkError 254 "Could not move output to $GATHER_DIR"
	popd
}

# Move files from GATHER_DIR to OUTBOX
# Note that this function modifies $IFS
cdm_gather_export() {
	RESULT=1
	pushd ${GATHER_DIR}
	IFS="
"
	FILES=( $( ls -A -I ".cdm*.lock" -I ".cdm-outgoing-*" ) )
	log "FILES: ${#FILES[@]}"
	mkdir -p $GATHER_MY_OUTBOX
	checkError 254 "Could not mkdir ${GATHER_MY_OUTBOX}"
	if (( ${#FILES} > 0 )); then
		mv ${FILES[@]} $GATHER_MY_OUTBOX
		checkError 254 "Could not move ${FILES[@]} to $GATHER_MY_OUTBOX"
		RESULT=0
	fi
	popd
	return $RESULT
}

# Release the GATHER_LOCKFILE
cdm_gather_lock_release() {
	logstate "LOCK_RELEASE $GATHER_LOCKFILE"
	unlink $GATHER_LOCKFILE
}

# Move files from (LFS) OUTBOX to (GFS) GATHER_TARGET
cdm_gather_flush() {
	pushd ${GATHER_MY_OUTBOX}
	logstate "GATHER_TARGET $GATHER_TARGET"
	mkdir -p ${GATHER_TARGET}
	logstate "GATHER_FLUSH_START ${GATHER_TARGET}/cdm-gather-$ID.tar"
	TARBALL=${GATHER_TARGET}/cdm-gather-$ID.tar
	tar cf ${TARBALL} *
	checkError 254 "CDM[GATHER]: error writing: ${TARBALL}"
	logstate "GATHER_FLUSH_DONE"
	popd
}

# Called by _swiftwrap at the end of each job
cdm_gather_action() {
	GATHER_OUTPUT=${*}

	GATHER_DIR=$(    perl shared/cdm.pl property GATHER_DIR    < $CDM_FILE )
	GATHER_MAX=$(    perl shared/cdm.pl property GATHER_LIMIT  < $CDM_FILE )
	GATHER_TARGET=$( perl shared/cdm.pl property GATHER_TARGET < $CDM_FILE )

	cdm_gather_setup $GATHER_DIR
	cdm_gather_lock_acquire

	cdm_gather_import

	USAGE=$( du -s --exclude=".cdm*" -B 1 $GATHER_DIR )
	USAGE=${USAGE%	*} # Chop off filename in output

	logstate "USAGE_CHECK $USAGE / $GATHER_MAX"

	FLUSH="no"
	if (( USAGE > GATHER_MAX )); then
		FLUSH="yes"
		cdm_gather_export
	fi

	cdm_gather_lock_release
	if [ $FLUSH == "yes" ]; then
		cdm_gather_flush
	fi
}

# Called by cdm_cleanup.sh at the end of the workflow
cdm_gather_cleanup() {
	declare -p PWD DIR GATHER_DIR GATHER_TARGET ID
	cdm_gather_setup $GATHER_DIR
	cdm_gather_lock_acquire
	cdm_gather_export
	EXPORT_RESULT=$?
	cdm_gather_lock_release
	if (( EXPORT_RESULT == 0)); then
		cdm_gather_flush
	fi
}


# Local Variables:
# mode: sh
# sh-basic-offset: 4
# tab-width: 4
# indent-tabs-mode: 1
# End:
<commit_msg>Correct the argument handling for CDM LOCAL file action.<commit_after>
# Source this for CDM shell functions

log "Reading cdm_lib.sh ..."

# Do a CDM lookup
cdm_lookup() {
	CDM_PL=$1
	CDM_FILE=$2
	FILE=$3

	RESULT="DEFAULT"
	if [ ! -z $CDM_FILE ] && [ -f $CDM_PL ] && [ -f $CDM_FILE ]; then
		RESULT=$( perl $CDM_PL lookup $FILE < $CDM_FILE 2> $INFO )
		checkError 254 "cdm_lookup(): failed! (malformed CDM file?)"
	fi
	echo $RESULT
}

cdm_action() {
	log "CDM_ACTION: $@"

	local JOBDIR=$1  # Given jobdir
	local MODE=$2    # INPUT or OUTPUT
	local FILE=$3    # User file
	local POLICY=$4  # DIRECT, BROADCAST, ...
	shift 4
	local ARGS=$@

	log "POLICY=$POLICY"
	case $POLICY in
		DIRECT)
			DIRECT_DIR=${ARGS[0]}
			log "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE via $JOBDIR/$FILE"
			if [ $MODE == "INPUT" ]; then
				[ -f "$DIRECT_DIR/$FILE" ]
				checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!"
				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
			elif [ $MODE == "OUTPUT" ]; then
				mkdir -p $DIRECT_DIR
				checkError 254 "CDM[DIRECT]: mkdir -p $DIRECT_DIR failed!"
				mkdir -p $( dirname $DIRECT_DIR/$FILE )
				checkError 254 "CDM[DIRECT]: mkdir -p $( dirname $FILE ) failed!"
				touch $DIRECT_DIR/$FILE
				checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!"
				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
			else
				fail 254 "Unknown MODE: $MODE"
			fi
			;;
 		LOCAL)
 			#TOOL=${ARGS[0]}
 			#REMOTE_DIR=${ARGS[1]}
 			#FLAGS=${ARGS[3]}
			TOOL=$1
			REMOTE_DIR=$2
			FLAGS=$3
			log "CDM[LOCAL]: TOOL=$TOOL FLAGS=$FLAGS REMOTE_DIR=$REMOTE_DIR ARGS=$ARGS"
 			log "CDM[LOCAL]: Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE"
 			if [ $MODE == "INPUT" ]; then
 				[ -f "$REMOTE_DIR/$FILE" ]
 				checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!"
 				$TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
 				checkError 254 "CDM[LOCAL]: Tool failed!"
 			elif [ $MODE == "OUTPUT" ]; then
 				log "CDM[LOCAL]..."
 			else
 				fail 254 "Unknown MODE: $MODE"
 			fi
 			;;
		BROADCAST)
			BROADCAST_DIR=${ARGS[0]}
			if [ $MODE == "INPUT" ]; then
				log "CDM[BROADCAST]: Linking $JOBDIR/$FILE to $BROADCAST_DIR/$FILE"
				[ -f "$BROADCAST_DIR/$FILE" ]
				checkError 254 "CDM[BROADCAST]: $BROADCAST_DIR/$FILE does not exist!"
				ln -s $BROADCAST_DIR/$FILE $JOBDIR/$FILE
				checkError 254 "CDM[BROADCAST]: Linking to $BROADCAST_DIR/$FILE failed!"
			else
				echo "CDM[BROADCAST]: Skipping output file: ${FILE}"
			fi
			;;
		GATHER)
			if [ $MODE == "INPUT" ]; then
				fail 254 "Cannot GATHER an input file!"
			fi
    esac
}

# Setup GATHER_DIR and some variables
cdm_gather_setup() {
	GATHER_DIR=$1

	logstate "GATHER_DIR $GATHER_DIR"
	mkdir -p $GATHER_DIR
	checkError 254 "Could not create: $GATHER_DIR"

	GATHER_LOCKFILE=$GATHER_DIR/.cdm.lock
	GATHER_MY_FILE=$GATHER_DIR/.cdm-$ID.lock
	GATHER_MY_OUTBOX=$GATHER_DIR/.cdm-outgoing-$ID
}

# Acquire the GATHER_LOCKFILE
cdm_gather_lock_acquire() {
	TRYING=1
	COUNT=0

	touch $GATHER_MY_FILE
	checkError 254 "Could not touch my file: $GATHER_MY_FILE"
	logstate "LOCK_ACQUIRE $GATHER_MY_FILE"
	while (( TRYING )); do
		ln $GATHER_MY_FILE $GATHER_LOCKFILE
		CODE=$?
		TRYING=$CODE
		if (( TRYING )); then
			logstate "LOCK_DELAY $GATHER_MY_FILE : $CODE"
			if (( COUNT++ > 10 )); then
				fail 254 "Could not acquire lock!"
				exit 1
			fi
			sleep $(( RANDOM % 10 ))
		fi
	done
}

# Move files from JOBDIR to GATHER_DIR
cdm_gather_import() {
	pushd jobs/$JOBDIR/$ID
	logstate "GATHER_IMPORT $GATHER_OUTPUT"
	mv $GATHER_OUTPUT $GATHER_DIR
	checkError 254 "Could not move output to $GATHER_DIR"
	popd
}

# Move files from GATHER_DIR to OUTBOX
# Note that this function modifies $IFS
cdm_gather_export() {
	RESULT=1
	pushd ${GATHER_DIR}
	IFS="
"
	FILES=( $( ls -A -I ".cdm*.lock" -I ".cdm-outgoing-*" ) )
	log "FILES: ${#FILES[@]}"
	mkdir -p $GATHER_MY_OUTBOX
	checkError 254 "Could not mkdir ${GATHER_MY_OUTBOX}"
	if (( ${#FILES} > 0 )); then
		mv ${FILES[@]} $GATHER_MY_OUTBOX
		checkError 254 "Could not move ${FILES[@]} to $GATHER_MY_OUTBOX"
		RESULT=0
	fi
	popd
	return $RESULT
}

# Release the GATHER_LOCKFILE
cdm_gather_lock_release() {
	logstate "LOCK_RELEASE $GATHER_LOCKFILE"
	unlink $GATHER_LOCKFILE
}

# Move files from (LFS) OUTBOX to (GFS) GATHER_TARGET
cdm_gather_flush() {
	pushd ${GATHER_MY_OUTBOX}
	logstate "GATHER_TARGET $GATHER_TARGET"
	mkdir -p ${GATHER_TARGET}
	logstate "GATHER_FLUSH_START ${GATHER_TARGET}/cdm-gather-$ID.tar"
	TARBALL=${GATHER_TARGET}/cdm-gather-$ID.tar
	tar cf ${TARBALL} *
	checkError 254 "CDM[GATHER]: error writing: ${TARBALL}"
	logstate "GATHER_FLUSH_DONE"
	popd
}

# Called by _swiftwrap at the end of each job
cdm_gather_action() {
	GATHER_OUTPUT=${*}

	GATHER_DIR=$(    perl shared/cdm.pl property GATHER_DIR    < $CDM_FILE )
	GATHER_MAX=$(    perl shared/cdm.pl property GATHER_LIMIT  < $CDM_FILE )
	GATHER_TARGET=$( perl shared/cdm.pl property GATHER_TARGET < $CDM_FILE )

	cdm_gather_setup $GATHER_DIR
	cdm_gather_lock_acquire

	cdm_gather_import

	USAGE=$( du -s --exclude=".cdm*" -B 1 $GATHER_DIR )
	USAGE=${USAGE%	*} # Chop off filename in output

	logstate "USAGE_CHECK $USAGE / $GATHER_MAX"

	FLUSH="no"
	if (( USAGE > GATHER_MAX )); then
		FLUSH="yes"
		cdm_gather_export
	fi

	cdm_gather_lock_release
	if [ $FLUSH == "yes" ]; then
		cdm_gather_flush
	fi
}

# Called by cdm_cleanup.sh at the end of the workflow
cdm_gather_cleanup() {
	declare -p PWD DIR GATHER_DIR GATHER_TARGET ID
	cdm_gather_setup $GATHER_DIR
	cdm_gather_lock_acquire
	cdm_gather_export
	EXPORT_RESULT=$?
	cdm_gather_lock_release
	if (( EXPORT_RESULT == 0)); then
		cdm_gather_flush
	fi
}


# Local Variables:
# mode: sh
# sh-basic-offset: 4
# tab-width: 4
# indent-tabs-mode: 1
# End:

{'fraction_non_alphanumeric': 0.13397586176955806, 'fraction_numerical': 0.016410523573847355, 'mean_word_length': 3.9159197609901835, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7347056', 'n_tokens_mistral': 5249, 'n_tokens_neox': 4706, 'n_words': 1137}