<commit_before>#!/usr/bin/env bash

name="$(basename "$1" .csv)"
tmp="$(mktemp "pa-plot-$name.XXXXXX")"

cat "$1" \
  | sed 's/PA-\([^-]*\)-[^,]*,\(.*\)/\1 \2/g' \
  | sort \
  | uniq \
  | grep -v '[a-z]' \
  > "$tmp"

gnuplot <<EOF
set title "$name"
set timefmt "%y%m%d"
set xdata time
set terminal pngcairo dashed
set output "$name.png"
unset label
set offset graph 0.01, graph 0.01, graph 0.05, graph 0.05
plot "$tmp" using 1:2 notitle \
        with points pt 5 lc rgb "#00aa00", \
     "$tmp" using 1:2 notitle smooth csplines \
        with lines lt 1 dt "-.-" lc rgb "#888888" lw 2
EOF

if [ $? -ne 0 ]; then
  echo "Failed to plot. $1 contained:"
  cat "$1"
  echo
  echo "Massaged input to gnuplot was:"
  cat "$tmp"
  echo
fi

rm "$tmp"<commit_msg>PA: trying more verbose gnuplot<commit_after>#!/usr/bin/env bash

name="$(basename "$1" .csv)"
tmp="$(mktemp "pa-plot-$name.XXXXXX")"

cat "$1" \
  | sed 's/PA-\([^-]*\)-[^,]*,\(.*\)/\1 \2/g' \
  | sort \
  | uniq \
  | grep -v '[a-z]' \
  > "$tmp"

gnuplot <<EOF
set title "$name"
set timefmt "%y%m%d"
set xdata time
set terminal pngcairo dashed
set output "$name.png"
unset label
set xtics rotate
set offset graph 0.01, graph 0.01, graph 0.05, graph 0.05
plot "$tmp" using 1:2 notitle with points \
        pointtype 5 \
        linecolor rgb "#00aa00", \
     "$tmp" using 1:2 notitle smooth csplines with lines \
        linetype 2 \
        dashtype "-.-" \
        linecolor rgb "#888888" \
        linewidth 2
EOF

if [ $? -ne 0 ]; then
  echo "Failed to plot. $1 contained:"
  cat "$1"
  echo
  echo "Massaged input to gnuplot was:"
  cat "$tmp"
  echo
fi

rm "$tmp"
{'fraction_non_alphanumeric': 0.1855480710349051, 'fraction_numerical': 0.04409063074096754, 'mean_word_length': 2.5521739130434784, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '194161', 'n_tokens_mistral': 710, 'n_tokens_neox': 644, 'n_words': 210}