<commit_before>#! /bin/bash

### Naming Convention
# (WXYZ)_(  )_(  )_(abcd)
#   |     ||   ||   ||||
# OBJECT  ||   ||   ||||
# TYPE    ||   ||   ||||
#        SIZE  ||   ||||
#             COUNT ||||
#                 STRUCTURE

### OBJECT TYPE
JS="W"
CSS="X"
IMG="Y"
RND="Z"

### SIZES (in bytes)
unset SIZES
SIZES=(100000 250000 500000 750000 1000000 1500000 2000000 2500000 4000000 6000000 8000000)

### OBJECT COUNTS
unset OBJECTS
OBJECTS=(1 2 3 4 5 6 7 8 9 10 15 20 25 30 35 50 70 90 100 125 150 175 200)

### OBJECT STRUCTURE
SS="a" #All same size
ASC="b" #Ascending
DESC="c" #Descending
RNDS="d" #Random structure

### LOOP VARIBLES
OBJECT_STRUCTURE=0
SIZE=0
COUNT=0
OBJECT_TYPE=0
BS=1
WEBS0=""
WEBS1=""
WEBS2=""
WEBS3=""

### GENERATION
for i in `seq 0 3`; #OBJECT TYPE
do
        case "$i" in
        0)
                OBJECT_TYPE=0
                WEBS0=$JS
        ;;
        1)
                OBJECT_TYPE=1
                WEBS0=$CSS
        ;;
        2)
                OBJECT_TYPE=2
                WEBS0=$IMG
        ;;
        3)
                OBJECT_TYPE=4
                WEBS0=$RND
        ;;
        esac
        for j in ${!SIZES[@]}; #SIZE
        do
		WEBS1=$j
		SIZE=${SIZES[$j]}
		
		if [[ "$SIZE" -gt "999999" ]]
		then
			BS=100
		else
			BS=1
		fi

                for k in ${!OBJECTS[@]}; #OBJECT COUNT
                do
			WEBS2=$k
			COUNT=${OBJECTS[$k]}
                        for l in `seq 0 3`; #OBJECT STRUCTURE
                        do
				case "$l" in
				0)
					OBJECT_STRUCTURE=0
					WEBS3=$SS
				;;
				1)
					OBJECT_STRUCTURE=1
					WEBS3=$ASC
				;;
				2)
					OBJECT_STRUCTURE=2
					WEBS3=$DESC
				;;
				3)
					OBJECT_STRUCTURE=3
					WEBS3=$RNDS
				;;
				esac

				### ACTUAL GENERATION
				./website_gen.sh $COUNT $OBJECT_STRUCTURE $OBJECT_TYPE 0 $SIZE $BS $WEBS0"_"$WEBS1"_"$WEBS2"_"$WEBS3
				#echo $WEBS0"_"$WEBS1"_"$WEBS2"_"$WEBS3"_"$COUNT"_"$SIZE
				
                        done # /OBJECT STRUCTURE
                done # /COUNT
        done # /SIZE
done # /TYPE

#
# Used print out size of result constants
#
echo -n "const SIZE = ["
for i in ${SIZES[@]};
do

	if [[ "$i" -lt "999999" ]]
	then
		BASE=1000
		UNITS="KB"
	else
		BASE=1000000
		UNITS="MB"
	fi
	
	#SPENCER REGEX HERE
	VALUE=$(python -c "print float($i) / float($BASE)")

	if [ "$i" != "${SIZES[-1]}" ]
	then
		echo -n '"'$VALUE $UNITS'"'," "
	else
		echo '"'$VALUE $UNITS'"'"];"<commit_msg>added regex for .0 vs .5<commit_after>#! /bin/bash

### Naming Convention
# (WXYZ)_(  )_(  )_(abcd)
#   |     ||   ||   ||||
# OBJECT  ||   ||   ||||
# TYPE    ||   ||   ||||
#        SIZE  ||   ||||
#             COUNT ||||
#                 STRUCTURE

### OBJECT TYPE
JS="W"
CSS="X"
IMG="Y"
RND="Z"

### SIZES (in bytes)
unset SIZES
SIZES=(100000 250000 500000 750000 1000000 1500000 2000000 2500000 4000000 6000000 8000000)

### OBJECT COUNTS
unset OBJECTS
OBJECTS=(1 2 3 4 5 6 7 8 9 10 15 20 25 30 35 50 70 90 100 125 150 175 200)

### OBJECT STRUCTURE
SS="a" #All same size
ASC="b" #Ascending
DESC="c" #Descending
RNDS="d" #Random structure

### LOOP VARIBLES
OBJECT_STRUCTURE=0
SIZE=0
COUNT=0
OBJECT_TYPE=0
BS=1
WEBS0=""
WEBS1=""
WEBS2=""
WEBS3=""

### GENERATION
for i in `seq 0 3`; #OBJECT TYPE
do
        case "$i" in
        0)
                OBJECT_TYPE=0
                WEBS0=$JS
        ;;
        1)
                OBJECT_TYPE=1
                WEBS0=$CSS
        ;;
        2)
                OBJECT_TYPE=2
                WEBS0=$IMG
        ;;
        3)
                OBJECT_TYPE=4
                WEBS0=$RND
        ;;
        esac
        for j in ${!SIZES[@]}; #SIZE
        do
		WEBS1=$j
		SIZE=${SIZES[$j]}
		
		if [[ "$SIZE" -gt "999999" ]]
		then
			BS=100
		else
			BS=1
		fi

                for k in ${!OBJECTS[@]}; #OBJECT COUNT
                do
			WEBS2=$k
			COUNT=${OBJECTS[$k]}
                        for l in `seq 0 3`; #OBJECT STRUCTURE
                        do
				case "$l" in
				0)
					OBJECT_STRUCTURE=0
					WEBS3=$SS
				;;
				1)
					OBJECT_STRUCTURE=1
					WEBS3=$ASC
				;;
				2)
					OBJECT_STRUCTURE=2
					WEBS3=$DESC
				;;
				3)
					OBJECT_STRUCTURE=3
					WEBS3=$RNDS
				;;
				esac

				### ACTUAL GENERATION
				./website_gen.sh $COUNT $OBJECT_STRUCTURE $OBJECT_TYPE 0 $SIZE $BS $WEBS0"_"$WEBS1"_"$WEBS2"_"$WEBS3
				#echo $WEBS0"_"$WEBS1"_"$WEBS2"_"$WEBS3"_"$COUNT"_"$SIZE
				
                        done # /OBJECT STRUCTURE
                done # /COUNT
        done # /SIZE
done # /TYPE

#
# Used print out size of result constants
#
echo -n "const SIZE = ["
for i in ${SIZES[@]};
do

	if [[ "$i" -lt "999999" ]]
	then
		BASE=1000
		UNITS="KB"
	else
		BASE=1000000
		UNITS="MB"
	fi
	
	VALUE=$(python -c "print float($i) / float($BASE)" | sed -e 's/\.0//g')

	if [ "$i" != "${SIZES[-1]}" ]
	then
		echo -n '"'$VALUE $UNITS'"'," "
	else
		echo '"'$VALUE $UNITS'"'"];"
{'fraction_non_alphanumeric': 0.15577476366625564, 'fraction_numerical': 0.07912042745581586, 'mean_word_length': 1.7684869169510808, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2462353', 'n_tokens_mistral': 2393, 'n_tokens_neox': 1853, 'n_words': 462}