<commit_before>#!/bin/bash
# 
# Copyright (c) 2015-2016, <NAME>. All rights reserved.
# 
# “Singularity” Copyright (c) 2016, The Regents of the University of California,
# through Lawrence Berkeley National Laboratory (subject to receipt of any
# required approvals from the U.S. Dept. of Energy).  All rights reserved.
# 
# This software is licensed under a customized 3-clause BSD license.  Please
# consult LICENSE file distributed with the sources of this project regarding
# your rights to use or distribute this software.
# 
# NOTICE.  This Software was developed under funding from the U.S. Department of
# Energy and the U.S. Government consequently retains certain rights. As such,
# the U.S. Government has been granted for itself and others acting on its
# behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
# to reproduce, distribute copies to the public, prepare derivative works, and
# perform publicly and display publicly, and to permit other to do so. 
# 
# 


## Basic sanity
if [ -z "$SINGULARITY_libexecdir" ]; then
    echo "Could not identify the Singularity libexecdir."
    exit 1
fi

## Load functions
if [ -f "$SINGULARITY_libexecdir/singularity/functions" ]; then
    . "$SINGULARITY_libexecdir/singularity/functions"
else
    echo "Error loading functions: $SINGULARITY_libexecdir/singularity/functions"
    exit 1
fi


if [ -z "${SINGULARITY_IMAGE:-}" ]; then
    message ERROR "SINGULARITY_IMAGE is undefined...\n"
    ABORT 255
fi

if [ -z "${SINGULARITY_COMMAND:-}" ]; then
    message ERROR "SINGULARITY_COMMAND is undefined...\n"
    ABORT 255
fi


# Don't trust environment
USERID=`id -u`
HOMEDIR=`getent passwd $USERID | cut -d: -f6`

case "$SINGULARITY_IMAGE" in
    *.tgz|*.tar.gz)
        NAME=`basename "$SINGULARITY_IMAGE"`
        TIMESTAMP=`stat -c "%Y" "$SINGULARITY_IMAGE"`
        CONTAINER_DIR="$HOMEDIR/.singularity/cache/$NAME/$TIMESTAMP/$NAME"
        if [ ! -d "$CONTAINER_DIR" ]; then
            mkdir -p "$CONTAINER_DIR"
            tar -C "$CONTAINER_DIR" -xzf "$SINGULARITY_IMAGE" 2>/dev/null # this almost always gives permission errors
        fi
        SINGULARITY_IMAGE="$CONTAINER_DIR"
    ;;
esac



if [ -x "$SINGULARITY_libexecdir/singularity/sexec" ]; then
    exec "$SINGULARITY_libexecdir/singularity/sexec" "$@" <&0
else
    message ERROR "Could not locate the Singularity Exec module: $SEXEC\n"
    exit 1
fi<commit_msg>Initial functionality of downloading image from URL<commit_after>#!/bin/bash
# 
# Copyright (c) 2015-2016, <NAME>. All rights reserved.
# 
# “Singularity” Copyright (c) 2016, The Regents of the University of California,
# through Lawrence Berkeley National Laboratory (subject to receipt of any
# required approvals from the U.S. Dept. of Energy).  All rights reserved.
# 
# This software is licensed under a customized 3-clause BSD license.  Please
# consult LICENSE file distributed with the sources of this project regarding
# your rights to use or distribute this software.
# 
# NOTICE.  This Software was developed under funding from the U.S. Department of
# Energy and the U.S. Government consequently retains certain rights. As such,
# the U.S. Government has been granted for itself and others acting on its
# behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
# to reproduce, distribute copies to the public, prepare derivative works, and
# perform publicly and display publicly, and to permit other to do so. 
# 
# 


## Basic sanity
if [ -z "$SINGULARITY_libexecdir" ]; then
    echo "Could not identify the Singularity libexecdir."
    exit 1
fi

## Load functions
if [ -f "$SINGULARITY_libexecdir/singularity/functions" ]; then
    . "$SINGULARITY_libexecdir/singularity/functions"
else
    echo "Error loading functions: $SINGULARITY_libexecdir/singularity/functions"
    exit 1
fi


if [ -z "${SINGULARITY_IMAGE:-}" ]; then
    message ERROR "SINGULARITY_IMAGE is undefined...\n"
    ABORT 255
fi

if [ -z "${SINGULARITY_COMMAND:-}" ]; then
    message ERROR "SINGULARITY_COMMAND is undefined...\n"
    ABORT 255
fi


# Don't trust environment
USERID=`id -u`
HOMEDIR=`getent passwd $USERID | cut -d: -f6`

case "$SINGULARITY_IMAGE" in
    http://*|https://*)
        NAME=`basename "$SINGULARITY_IMAGE"`
        if [ -f "$NAME" ]; then
            message 2 "Using cached container in current working directory: $NAME\n"
            SINGULARITY_IMAGE="$NAME"
        else
            message 1 "Caching container to current working directory: $NAME\n"
            if curl -L -k "$SINGULARITY_IMAGE" > "$NAME"; then
                SINGULARITY_IMAGE="$NAME"
            else
                ABORT 255
            fi
        fi
    ;;
esac

case "$SINGULARITY_IMAGE" in
    *.tgz|*.tar.gz)
        NAME=`basename "$SINGULARITY_IMAGE"`
        TIMESTAMP=`stat -c "%Y" "$SINGULARITY_IMAGE"`
        CONTAINER_DIR="$HOMEDIR/.singularity/cache/$NAME/$TIMESTAMP/$NAME"
        if [ ! -d "$CONTAINER_DIR" ]; then
            mkdir -p "$CONTAINER_DIR"
            tar -C "$CONTAINER_DIR" -xzf "$SINGULARITY_IMAGE" 2>/dev/null # this almost always gives permission errors
        fi
        SINGULARITY_IMAGE="$CONTAINER_DIR"
    ;;
    *.tbz|*.tar.bz)
        NAME=`basename "$SINGULARITY_IMAGE"`
        TIMESTAMP=`stat -c "%Y" "$SINGULARITY_IMAGE"`
        CONTAINER_DIR="$HOMEDIR/.singularity/cache/$NAME/$TIMESTAMP/$NAME"
        if [ ! -d "$CONTAINER_DIR" ]; then
            mkdir -p "$CONTAINER_DIR"
            tar -C "$CONTAINER_DIR" -xjf "$SINGULARITY_IMAGE" 2>/dev/null # this almost always gives permission errors
        fi
        SINGULARITY_IMAGE="$CONTAINER_DIR"
    ;;
esac



if [ -x "$SINGULARITY_libexecdir/singularity/sexec" ]; then
    exec "$SINGULARITY_libexecdir/singularity/sexec" "$@" <&0
else
    message ERROR "Could not locate the Singularity Exec module: $SEXEC\n"
    exit 1
fi
{'fraction_non_alphanumeric': 0.10440122824974411, 'fraction_numerical': 0.009553053565336063, 'mean_word_length': 3.2547169811320753, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4367345', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1749, 'n_words': 599}