<commit_before>#!/usr/bin/env zsh

dbname=freecoin

txeval() {
    mongo $dbname -eval 'db.transactions.find().forEach(printjson)' \
        | sed -e 's/,//g ; s/NumberLong//g ; s/(//g ; s/)//g' \
        | awk '

/amount" :/ { amount=$3 }
/to-id" :/ { recipient=$3 }
/from-id" :/ { sender=$3 }
/^}/ { print "txarr+=(" sender ":" recipient ":" amount ")" }
'}

typeset -a txarr
eval `txeval`

typeset -aU participants
for i in ${txarr}; do
    participants+=( ${i[(ws@:@)1]} )
    participants+=( ${i[(ws@:@)2]} )
done

print "${#txarr} transactions parsed"
print "${#participants} unique participants"

graphviz=`cat << EOF
digraph $dbname {\n
graph [ splines=compound, overlap=false, overlap_shrink=true, ranksep=3, pack=true, packmode=nodes, resolution=120 ];\n
node [shape=ellipse, style=filled];\n
EOF`

for p in ${participants}; do
graphviz+="$p\n"
done

for t in $txarr; do
    sender=${t[(ws@:@)1]}
    recipient=${t[(ws@:@)2]}
    amount=${t[(ws@:@)3]}
    graphviz+="$sender -> $recipient [weight=$amount]\n"
done
graphviz+="}\n"

print $graphviz > $dbname.dot
cat $dbname.dot | twopi -Tpng -o $dbname.png<commit_msg>demo gephi export for graph<commit_after>#!/usr/bin/env zsh

dbname=freecoin
# targets: gephi or graphviz
target=${1:-gephi}

txeval() {
    mongo $dbname -eval 'db.transactions.find().forEach(printjson)' \
        | sed -e 's/,//g ; s/NumberLong//g ; s/(//g ; s/)//g' \
        | awk '

/amount" :/ { amount=$3 }
/to-id" :/ { recipient=$3 }
/from-id" :/ { sender=$3 }
/^}/ { print "txarr+=(" sender ":" recipient ":" amount ")" }
'}

typeset -a txarr
eval `txeval`

typeset -aU participants
for i in ${txarr}; do
    participants+=( ${i[(ws@:@)1]} )
    participants+=( ${i[(ws@:@)2]} )
done

print "exporting to $target format"
print "${#txarr} transactions parsed"
print "${#participants} unique participants"

case $target in

    gephi)
        print "Source,Target,Weight" > $dbname.csv
        for t in $txarr; do
            sender=${t[(ws@:@)1]}
            recipient=${t[(ws@:@)2]}
            amount=${t[(ws@:@)3]}
            print "$sender,$recipient,$amount" >> $dbname.csv
        done
    ;;
    graphviz)

        graphviz=`cat << EOF
digraph $dbname {\n
graph [ splines=compound, overlap=false, overlap_shrink=true, ranksep=3, pack=true, packmode=nodes, resolution=120 ];\n
node [shape=ellipse, style=filled];\n
EOF`

        for p in ${participants}; do
            graphviz+="$p\n"
        done

        for t in $txarr; do
            sender=${t[(ws@:@)1]}
            recipient=${t[(ws@:@)2]}
            amount=${t[(ws@:@)3]}
            graphviz+="$sender -> $recipient [weight=$amount]\n"
        done
        graphviz+="}\n"

        print $graphviz > $dbname.dot
        cat $dbname.dot | twopi -Tpng -o $dbname.png
        ;;
esac
{'fraction_non_alphanumeric': 0.20646319569120286, 'fraction_numerical': 0.010053859964093357, 'mean_word_length': 2.9461756373937678, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5777982', 'n_tokens_mistral': 1066, 'n_tokens_neox': 1048, 'n_words': 232}