<commit_before>#!/bin/bash

OUTPUT_BASE="$1" # A base name

BASE_DIR=`dirname "$BASH_SOURCE"`"/../../.."

OUTPUT_FILE="$OUTPUT_BASE.csv"
OUTPUT_REL_FILE="$OUTPUT_BASE-relative.csv"

EJ_EXE="$BASE_DIR/ej"

CACHE_DIR="$BASE_DIR/boot-test-cache"

# Special cache location:
export ERJ_CACHE_DIR="$CACHE_DIR"

function error() {
    echo "*** $1" >&2
    exit 1
}

function measure() {
    local outfile="$1"; shift # Rest of arguments are passed to ej
    /usr/bin/time --format '%e\t%U\t%S\t%M' --output boot-stats.tmp1 \
      ./ej "$@" -noshell -sasl sasl_error_logger false -eval \
	'AA=hd([V || {T,_,V}<-erlang:system_info(allocated_areas), (T=='"'non_heap:Perm Gen'"' orelse T=='"'non_heap:PS Perm Gen'"')]),
         GHS=erlang:system_info(global_heaps_size),
         io:format("~b\t~b\n", [AA,GHS]),
         erlang:halt().' \
	< /dev/null > boot-stats.tmp2<commit_msg>Do GC before doing memory size measurement<commit_after>#!/bin/bash

OUTPUT_BASE="$1" # A base name

BASE_DIR=`dirname "$BASH_SOURCE"`"/../../.."

OUTPUT_FILE="$OUTPUT_BASE.csv"
OUTPUT_REL_FILE="$OUTPUT_BASE-relative.csv"

EJ_EXE="$BASE_DIR/ej"

CACHE_DIR="$BASE_DIR/boot-test-cache"

# Special cache location:
export ERJ_CACHE_DIR="$CACHE_DIR"

function error() {
    echo "*** $1" >&2
    exit 1
}

function measure() {
    local outfile="$1"; shift # Rest of arguments are passed to ej
    /usr/bin/time --format '%e\t%U\t%S\t%M' --output boot-stats.tmp1 \
      ./ej "$@" -noshell -sasl sasl_error_logger false -eval \
	'erlang:garbage_collect(),
         AA=hd([V || {T,_,V}<-erlang:system_info(allocated_areas), (T=='"'non_heap:Perm Gen'"' orelse T=='"'non_heap:PS Perm Gen'"')]),
         GHS=erlang:system_info(global_heaps_size),
         io:format("~b\t~b\n", [AA,GHS]),
         erlang:halt().' \
	< /dev/null > boot-stats.tmp2
{'fraction_non_alphanumeric': 0.1986754966887417, 'fraction_numerical': 0.00772626931567329, 'mean_word_length': 3.8605898123324396, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6857324', 'n_tokens_mistral': 721, 'n_tokens_neox': 704, 'n_words': 126}