<commit_before>#!/bin/bash
# Daily API usage reports. To be run as a cronjob at the end of the day.
# dnaber, 2015-11-14

LANG="de_DE.UTF-8"
DATE1=`date +"%Y%m%d"`
DATE2=`date +"%Y-%m-%d"`
# for testing:
#DATE1=20160331
#DATE2=2016-03-31
TMPFILE=/tmp/log.temp
TMPFILE_ALL=/tmp/log-all.temp
OUTFILE=/tmp/statusmail.txt

echo "Daily LanguageTool API Report $DATE2" >$OUTFILE
echo "" >>$OUTFILE

grep -h "$DATE2 " log-[0-9]-$DATE1*.txt log-1.txt log-2.txt >$TMPFILE
cat log-[0-9]-$DATE1*.txt log-1.txt log-2.txt >$TMPFILE_ALL

TOTAL=`grep -c "Check done:" $TMPFILE`
printf "Total text checks : %'d\n" $TOTAL >>$OUTFILE

TOTALHOME=`grep "Check done:" $TMPFILE | grep -c languagetool.org`
printf "Checks from lt.org: %'d\n" $TOTALHOME >>$OUTFILE

CHROME=`grep -c "chrome-extension" $TMPFILE`
printf "Chrome Requests   : %'d\n" $CHROME >>$OUTFILE

WEBEXTFF=`grep -c "webextension-firefox" $TMPFILE`
printf "WebExtension FF   : %'d\n" $WEBEXTFF >>$OUTFILE

WEBEXTCHROME=`grep -c "webextension-chrome" $TMPFILE`
printf "WebExtension Chr. : %'d\n" $WEBEXTCHROME >>$OUTFILE

ANDROID=`grep -c "androidspell" $TMPFILE`
printf "Android Requests  : %'d\n" $ANDROID >>$OUTFILE

CLIENT=`grep -c "java-http-client" $TMPFILE`
printf "Java Client Req   : %'d\n" $CLIENT >>$OUTFILE

GOOGLEAPP=`grep -c ":googledocs" $TMPFILE`
printf "Google Docs       : %'d\n" $GOOGLEAPP >>$OUTFILE

SUBLIME=`grep -c ":sublime" $TMPFILE`
printf "Sublime Requests  : %'d\n" $SUBLIME >>$OUTFILE

MSWORD=`grep -c ":msword" $TMPFILE`
printf "MS-Word Requests  : %'d\n" $MSWORD >>$OUTFILE

echo "$DATE2;$TOTAL;$FF;$CHROME;$ANDROID;$CLIENT;$SUBLIME;$WEBEXT;$MSWORD" >>/home/languagetool/api/api-log.csv

echo "" >>$OUTFILE
echo "An error has occurred      : `grep -c 'An error has occurred' $TMPFILE`" >>$OUTFILE
echo "too many parallel requests : `grep -c 'too many parallel requests' $TMPFILE`" >>$OUTFILE
echo "too many requests          : `grep -c 'too many requests' $TMPFILE`" >>$OUTFILE
echo "too many requests (Android): `grep -c 'too many requests.*androidspell' $TMPFILE`" >>$OUTFILE
#echo "TextTooLongException : `grep -c 'TextTooLongException' $TMPFILE`" >>$OUTFILE
#echo "TimeoutException     : `grep -c 'java.util.concurrent.TimeoutException' $TMPFILE`" >>$OUTFILE

echo "Top HTTP error codes:" >>$OUTFILE
grep "An error has occurred" /tmp/log.temp|sed 's/.*HTTP code \([0-9]\+\)..*/HTTP code \1/'|sort |uniq -c| sort -r -n >>$OUTFILE

echo "" >>$OUTFILE<commit_msg>log number of incomplete results sent<commit_after>#!/bin/bash
# Daily API usage reports. To be run as a cronjob at the end of the day.
# dnaber, 2015-11-14

LANG="de_DE.UTF-8"
DATE1=`date +"%Y%m%d"`
DATE2=`date +"%Y-%m-%d"`
# for testing:
#DATE1=20160331
#DATE2=2016-03-31
TMPFILE=/tmp/log.temp
TMPFILE_ALL=/tmp/log-all.temp
OUTFILE=/tmp/statusmail.txt

echo "Daily LanguageTool API Report $DATE2" >$OUTFILE
echo "" >>$OUTFILE

grep -h "$DATE2 " log-[0-9]-$DATE1*.txt log-1.txt log-2.txt >$TMPFILE
cat log-[0-9]-$DATE1*.txt log-1.txt log-2.txt >$TMPFILE_ALL

TOTAL=`grep -c "Check done:" $TMPFILE`
printf "Total text checks : %'d\n" $TOTAL >>$OUTFILE

TOTALHOME=`grep "Check done:" $TMPFILE | grep -c languagetool.org`
printf "Checks from lt.org: %'d\n" $TOTALHOME >>$OUTFILE

CHROME=`grep -c "chrome-extension" $TMPFILE`
printf "Chrome Requests   : %'d\n" $CHROME >>$OUTFILE

WEBEXTFF=`grep -c "webextension-firefox" $TMPFILE`
printf "WebExtension FF   : %'d\n" $WEBEXTFF >>$OUTFILE

WEBEXTCHROME=`grep -c "webextension-chrome" $TMPFILE`
printf "WebExtension Chr. : %'d\n" $WEBEXTCHROME >>$OUTFILE

ANDROID=`grep -c "androidspell" $TMPFILE`
printf "Android Requests  : %'d\n" $ANDROID >>$OUTFILE

CLIENT=`grep -c "java-http-client" $TMPFILE`
printf "Java Client Req   : %'d\n" $CLIENT >>$OUTFILE

GOOGLEAPP=`grep -c ":googledocs" $TMPFILE`
printf "Google Docs       : %'d\n" $GOOGLEAPP >>$OUTFILE

SUBLIME=`grep -c ":sublime" $TMPFILE`
printf "Sublime Requests  : %'d\n" $SUBLIME >>$OUTFILE

MSWORD=`grep -c ":msword" $TMPFILE`
printf "MS-Word Requests  : %'d\n" $MSWORD >>$OUTFILE

echo "$DATE2;$TOTAL;$FF;$CHROME;$ANDROID;$CLIENT;$SUBLIME;$WEBEXT;$MSWORD" >>/home/languagetool/api/api-log.csv

echo "" >>$OUTFILE
echo "An error has occurred      : `grep -c 'An error has occurred' $TMPFILE`" >>$OUTFILE
echo "too many parallel requests : `grep -c 'too many parallel requests' $TMPFILE`" >>$OUTFILE
echo "too many requests          : `grep -c 'too many requests' $TMPFILE`" >>$OUTFILE
echo "too many requests (Android): `grep -c 'too many requests.*androidspell' $TMPFILE`" >>$OUTFILE
#echo "TextTooLongException : `grep -c 'TextTooLongException' $TMPFILE`" >>$OUTFILE
#echo "TimeoutException     : `grep -c 'java.util.concurrent.TimeoutException' $TMPFILE`" >>$OUTFILE
echo "Incomplete results sent    : `grep -c  "matches found so far" $TMPFILE`" >>$OUTFILE

echo "Top HTTP error codes:" >>$OUTFILE
grep "An error has occurred" /tmp/log.temp|sed 's/.*HTTP code \([0-9]\+\)..*/HTTP code \1/'|sort |uniq -c| sort -r -n >>$OUTFILE

echo "" >>$OUTFILE
{'fraction_non_alphanumeric': 0.1811623246492986, 'fraction_numerical': 0.018036072144288578, 'mean_word_length': 4.723623853211009, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3300691', 'n_tokens_mistral': 2087, 'n_tokens_neox': 2040, 'n_words': 520}