<commit_before>TITLE="Number of plays per day"
DB=openwhyd_dump
NAME=plot-nb-plays-per-day

echo "generate data from mongodb data ... (⚠️ may take several minutes)"
SECONDS=0
mongo --quiet $DB ./$NAME.mongo.js
echo ⏲ $SECONDS seconds.
# extract fields from resulting collection
keys=`mongo $DB --eval "rs.slaveOk();var keys = ['_id']; for(var key in db['$NAME'].find().sort({_id: -1}).limit(1)[0].value) { keys.push('value.' + key); }; keys.join(',');" --quiet`
# remove "value." prefixes from field names
columns=${keys//value./}
# generate csv file header, based on field names
echo $columns >$NAME.temp.csv
# count fields
nb_columns=`echo $columns | sed 's/[^,]//g' | wc -c`
# append resulting collection into output csv file
mongoexport -d $DB -c "$NAME" --type=csv --fields "$keys" | tail -n+2 >>$NAME.temp.csv
sed -i '' -e '$ d' $NAME.temp.csv # remove last line

echo "plot data to ../plots/$NAME.png ..."
mkdir ../plots &>/dev/null
gnuplot -c plot-csv-data.gp $NAME.temp.csv $nb_columns "$TITLE" >../plots/$NAME.png

echo "open ../plots/$NAME.png ..."
open ../plots/$NAME.png
# rm $NAME.temp.csv<commit_msg>data analysis: fixed «player» columns for nb-plays-per-day<commit_after>TITLE="Number of plays per day"
DB=openwhyd_dump
NAME=plot-nb-plays-per-day
FIELDS=_id,value.total,value.yt,value.sc,value.dm,value.vi,value.dz,value.ja,value.bc,value.fi,value.sp
COLUMNS="Date,Total plays,Youtube,SoundCloud,Dailymotion,Vimeo,Deezer,Jamendo,Bandcamp,Audio file,Spotify"
# from https://github.com/openwhyd/openwhyd/blob/d27fb71220cbd29e9e418bd767426e3b4a2187f3/whydJS/public/js/whydPlayer.js#L559
NB_COLUMNS=`echo $COLUMNS | sed 's/[^,]//g' | wc -c` # count COLUMNS

echo "generate data from mongodb data ... (⚠️ may take several minutes)"
SECONDS=0
mongo --quiet $DB ./$NAME.mongo.js
echo ⏲ $SECONDS seconds.
# write resulting collection into output csv file, with custom header row
echo $COLUMNS >$NAME.temp.csv
mongoexport -d $DB -c "$NAME" --type=csv --fields "$FIELDS" | tail -n+2 >>$NAME.temp.csv
sed -i '' -e '$ d' $NAME.temp.csv # remove last line

echo "plot data to ../plots/$NAME.png ..."
mkdir ../plots &>/dev/null
gnuplot -c plot-csv-data.gp $NAME.temp.csv $NB_COLUMNS "$TITLE" >../plots/$NAME.png

echo "open ../plots/$NAME.png ..."
open ../plots/$NAME.png
rm $NAME.temp.csv
{'fraction_non_alphanumeric': 0.18463553033609778, 'fraction_numerical': 0.01571366215626364, 'mean_word_length': 4.139013452914798, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1791402', 'n_tokens_mistral': 907, 'n_tokens_neox': 850, 'n_words': 224}