<commit_before>#!/bin/bash

# Usage:
# $ cd app
# $ ./topicStatsReport.sh
# $ ls ../../var/reporting
# Now you can view the report.

# # Assume this is run from dir of the script then change to app dir.
cd ../..

sqlite3 var/db.sqlite -csv -header \
  < lib/query/sql/topicStats.sql \
  > var/reporting/topic_report_$(date +"%y-%m-%d@%T").csv<commit_msg>Improve topic stats report generator with flexible options and better docs.<commit_after>#!/bin/bash
###############################################################################
#
# Generate the topic stats report with a query to the database and send
# the results out to a pre-configured CSV file for the current date.
#
# This script must be run with app as current working directory.
#
# Usage:
# $ cd app
# $ ./utils/reporting/topicStatsReport.sh [pathToDB]
#     where pathToDB is the configured db path. Defaults to 'var/db.sqlite'
#     if not supplied.
#
# $ view var/reporting/
# Now you can view the report, which starts with topic_report in the name
# and has the current date. Any existing report on the day will be overwritten.
#
###############################################################################


# Check for first argument otherwise use the default.
if [ ! -z "$1" ]
  then
    dbName=$1
  else
    dbName=db.sqlite
fi

echo DB path: $dbName

inPath=lib/query/sql/topicStats.sql
echo Report: $inPath

outPath=var/reporting/topic_report_$(date +"%y-%m-%d").csv

sqlite3 $dbName -csv -header < $inPath > $outPath && echo Output: $outPath
{'fraction_non_alphanumeric': 0.20893561103810776, 'fraction_numerical': 0.002628120893561104, 'mean_word_length': 3.977124183006536, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4540268', 'n_tokens_mistral': 473, 'n_tokens_neox': 452, 'n_words': 163}