<commit_before>#!/bin/sh
#gc.sh - run ecdump.

remove_if_nodiffs()
#Usage:  remove_if_nodiffs lastdumpdir newdumpdir
{
    #see if we can delete the previous dump.  we are looking for a ddiff file that
    #looks like this:
    #+-----------------------------------------------------------------+
    #|### In 1303151230 but not in 1303151330:                         |
    #|    NULL                                                         |
    #|### In 1303151330 but not in 1303151230:                         |
    #|    NULL                                                         |
    #|### In both 1303151230 and 1303151330:, with different content:  |
    #|    NULL                                                         |
    #+-----------------------------------------------------------------+

    #get last, curr run dirs:
    lastrun="$1"
    newrun="$2"

    echo remove_if_nodiffs: lastrun=$lastrun newrun=$newrun

    #note - first char is actually a tab, but grep does not allow \t.  RT 3/15/13
    nullcnt=`grep '^.NULL$' logs/$newrun/ddiff.log | wc -w`

    if [ "$nullcnt" -eq 3 ]; then
        bldmsg -p $p -mark New run has no changes, removing identical new dump: $newrun
        bldmsg -p $p -markbeg remove identical dump
        rm -rf "$newrun"
        status=$?
        bldmsg -p $p -status $status -markbeg remove identical dump
        if [ $status -ne  0 ]; then
            bldmsg -p $p -error -status $status command failed:  rm -rf "$newrun"
            exit 1
        fi
    else
        bldmsg -p $p -mark New run has changes - see logs/$newrun/ddiff.log for differences
    fi
}

p=`basename $0`
remove_if_nodiffs $*
<commit_msg>updates from doecdump.sh Committed on Fri Mar 15 17:22:36 PDT 2013<commit_after>#!/bin/sh
#gc.sh - run ecdump.

remove_if_nodiffs()
#Usage:  remove_if_nodiffs lastdumpdir newdumpdir
{
    #see if we can delete the new dump.  we are looking for a ddiff file that
    #looks like this:
    #+-----------------------------------------------------------------+
    #|### In 1303151230 but not in 1303151330:                         |
    #|    NULL                                                         |
    #|### In 1303151330 but not in 1303151230:                         |
    #|    NULL                                                         |
    #|### In both 1303151230 and 1303151330:, with different content:  |
    #|    NULL                                                         |
    #+-----------------------------------------------------------------+

    #get last, curr run dirs:
    lastrun="$1"
    newrun="$2"

    echo remove_if_nodiffs: lastrun=$lastrun newrun=$newrun

    #note - first char is actually a tab, but grep does not allow \t.  RT 3/15/13
    nullcnt=`grep '^.NULL$' logs/$newrun/ddiff.log | wc -w`
    _rind_status=0

    if [ "$nullcnt" -eq 3 ]; then
        bldmsg -p $p -mark New run has no changes, removing it: $newrun
        bldmsg -p $p -markbeg remove identical dump
        rm -rf "$newrun"
        _rind_status=$?
        bldmsg -p $p -status $_rind_status -markend remove identical dump
        if [ $_rind_status -ne  0 ]; then
            bldmsg -p $p -error -status $_rind_status command failed:  rm -rf "$newrun"
        fi
    else
        bldmsg -p $p -mark New run has changes - see logs/$newrun/ddiff.log for differences
    fi

    return $_rind_status
}

p=`basename $0`
remove_if_nodiffs $*

{'fraction_non_alphanumeric': 0.1668614845119813, 'fraction_numerical': 0.04500292226767972, 'mean_word_length': 1.2729083665338645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '53166', 'n_tokens_mistral': 1106, 'n_tokens_neox': 970, 'n_words': 323}