<commit_before>#!/bin/sh

if test "x$srcdir" = x ; then srcdir=`pwd`; fi 
. ../test_common.sh

. "$srcdir/test_nczarr.sh"

# This shell script tests support for the NC_STRING type

set -e

zarrscalar() {
    rm -f $2
    sed -e '/dimensions:/d' -e '/_scalar_ =/d' -e '/int v/ s|(_scalar_)||' <$1 >$2
}

testcase() {
zext=$1

echo "*** Test: scalar write/read"

# Get pure zarr args
fileargs tmp_scalar_zarr "mode=zarr,$zext"
zarrurl="$fileurl"
zarrfile="$file"
# Get nczarr args
fileargs tmp_scalar_nczarr "mode=nczarr,$zext"
nczarrurl="$fileurl"
nczarrfile="$file"

# setup
deletemap $zext $zarrfile
deletemap $zext $nczarrfile

# Create alternate ref files
echo "*** create pure zarr file"
${NCGEN} -4 -b -o "$zarrurl" $srcdir/ref_scalar.cdl
echo "*** create nczarr file"
${NCGEN} -4 -b -o "$nczarrurl" $srcdir/ref_scalar.cdl

echo "*** read purezarr"
${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr0_${zext}.cdl
${ZMD} -h $zarrurl > tmp_scalar_zarr_${zext}.txt
echo "*** read nczarr"
${NCDUMP} -n ref_scalar $nczarrurl > tmp_scalar_nczarr_${zext}.cdl
${ZMD} -h $nczarrurl > tmp_scalar_nczarr_${zext}.txt

echo "*** verify"
diff -bw ref_scalar.cdl tmp_scalar_nczarr_${zext}.cdl

# Fixup
zarrscalar tmp_scalar_zarr0_${zext}.cdl tmp_scalar_zarr_${zext}.cdl
diff -bw ref_scalar.cdl tmp_scalar_zarr_${zext}.cdl
}

testcase file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi

exit 0<commit_msg>test2<commit_after>#!/bin/sh

if test "x$srcdir" = x ; then srcdir=`pwd`; fi 
. ../test_common.sh

. "$srcdir/test_nczarr.sh"

# This shell script tests support for the NC_STRING type

set -x
set -e

zarrscalar() {
    rm -f $2
    sed -e '/dimensions:/d' -e '/_scalar_ =/d' -e '/int v/ s|(_scalar_)||' <$1 >$2
}

testcase() {
zext=$1

echo "*** Test: scalar write/read"

# Get pure zarr args
fileargs tmp_scalar_zarr "mode=zarr,$zext"
zarrurl="$fileurl"
zarrfile="$file"
# Get nczarr args
fileargs tmp_scalar_nczarr "mode=nczarr,$zext"
nczarrurl="$fileurl"
nczarrfile="$file"

# setup
deletemap $zext $zarrfile
deletemap $zext $nczarrfile

# Create alternate ref files
echo "*** create pure zarr file"
${NCGEN} -4 -b -o "$zarrurl" $srcdir/ref_scalar.cdl
echo "*** create nczarr file"
${NCGEN} -4 -b -o "$nczarrurl" $srcdir/ref_scalar.cdl

echo "*** read purezarr"
${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr0_${zext}.cdl
${ZMD} -h $zarrurl > tmp_scalar_zarr_${zext}.txt
echo "*** read nczarr"
${NCDUMP} -n ref_scalar $nczarrurl > tmp_scalar_nczarr_${zext}.cdl
${ZMD} -h $nczarrurl > tmp_scalar_nczarr_${zext}.txt

echo "*** verify"
diff -bw ${srcdir}/ref_scalar.cdl tmp_scalar_nczarr_${zext}.cdl

# Fixup
zarrscalar tmp_scalar_zarr0_${zext}.cdl tmp_scalar_zarr_${zext}.cdl
diff -bw ${srcdir}/ref_scalar.cdl tmp_scalar_zarr_${zext}.cdl
}

testcase file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi

exit 0
{'fraction_non_alphanumeric': 0.1587140439932318, 'fraction_numerical': 0.0077834179357022, 'mean_word_length': 4.865079365079365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3483478', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1293, 'n_words': 301}