<commit_before>#!/bin/bash -e

MY_DIR=$(cd $(dirname $BASH_SOURCE); pwd)

: ${2:?Usage: from_ts.sh (PATH|URL) TARGET.json}

readonly SOURCE=$1
readonly TARGET=$2

echo -n $TARGET

reportError() {
    echo " - error: $(cat | grep Error:)"
}

filterBadTypes() {
    node -pe '
        var json = JSON.parse(require("fs").readFileSync("'$TARGET.tmp'"));
        for (var p in json) {
            if (json[p]["!type"] && !/^(fn\(|\[)/.test(json[p]["!type"]))
                delete json[p]["!type"];
        }
        JSON.stringify(json, null, 2);
    '
}

if ! ERROR=$($MY_DIR/node_modules/tern/bin/from_ts "$SOURCE" 2>&1 | sed 's/\[object Object\]/?/g' >$TARGET.tmp); then
    rm -f $TARGET.tmp $TARGET
    echo "$ERROR" | reportError
else
    if ! node -e 'JSON.parse(require("fs").readFileSync("'$TARGET.tmp'").toString())' 2>/dev/null; then
        # Error reported with zero exit code
        cat $TARGET.tmp | reportError
        rm -f $TARGET.tmp $TARGET
        exit
    fi
    <commit_msg>Remove broken types recursively<commit_after>#!/bin/bash -e

MY_DIR=$(cd $(dirname $BASH_SOURCE); pwd)

: ${2:?Usage: from_ts.sh (PATH|URL) TARGET.json}

readonly SOURCE=$1
readonly TARGET=$2

echo -n $TARGET

reportError() {
    echo " - error: $(cat | grep Error:)"
}

filterBadTypes() {
    node -pe '
        var json = JSON.parse(require("fs").readFileSync("'$TARGET.tmp'"));
        function removeBadTypes(entry) {
            if (typeof entry === "string")
                return entry;
            for (var p in entry) {
                entry[p] = removeBadTypes(entry[p]);
                if (entry[p]["!type"] && !/^(fn\(|\[)/.test(entry[p]["!type"]))
                    delete entry[p]["!type"];
            }
            return entry;
        }
        JSON.stringify(removeBadTypes(json), null, 2);
    '
}

if ! ERROR=$($MY_DIR/node_modules/tern/bin/from_ts "$SOURCE" 2>&1 | sed 's/\[object Object\]/?/g' >$TARGET.tmp); then
    rm -f $TARGET.tmp $TARGET
    echo "$ERROR" | reportError
else
    if ! node -e 'JSON.parse(require("fs").readFileSync("'$TARGET.tmp'").toString())' 2>/dev/null; then
        # Error reported with zero exit code
        cat $TARGET.tmp | reportError
        rm -f $TARGET.tmp $TARGET
        exit
    fi
    
{'fraction_non_alphanumeric': 0.1845264562027568, 'fraction_numerical': 0.006224988883948421, 'mean_word_length': 2.4722222222222223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2670390', 'n_tokens_mistral': 790, 'n_tokens_neox': 762, 'n_words': 175}