<commit_before>#!/bin/sh
#dnaber, 2016-11-14

FIRSTPART=naber
LASTPART=danielnaber.de
TMPDATA=/tmp/lt_hourly_error_data.txt
LIMIT=0
DATE=`date +"%Y%m"`

cat log-[12]-${DATE}*_*.txt log-1.txt log-2.txt | grep "`date +"%Y-%m-%d %H:"`" | \
  egrep "An error has occurred" | \
  grep -v "Could not decode query" | \
  grep -v "URLDecoder: Illegal hex characters" | \
  grep -v "Missing 'language' parameter" | \
  grep -v "Text checking took longer than" | \
  grep -v "Missing 'text' or 'data' parameter" | \
  grep -v "is not a language code known" | \
  grep -v "Unsupported action: 'v2/check" | \
  grep -v "code 413" >$TMPDATA
#cat log-1.txt log-2.txt | grep "`date +"%Y-%m-%d %H:"`" | egrep "An error has occurred" | grep -v "code 413" >$TMPDATA
MATCHES=`cat $TMPDATA | wc -l`

if [ "$MATCHES" -gt "$LIMIT" ]
then
  cat $TMPDATA | tail -n 100 | \
        ifne mail -a 'Content-Type: text/plain; charset=utf-8' -s "Hourly LT Error Report ($MATCHES total)" ${FIRSTPART}@${LASTPART}
fi<commit_msg>improve mail filtering<commit_after>#!/bin/sh
#dnaber, 2016-11-14

FIRSTPART=naber
LASTPART=danielnaber.de
TMPDATA=/tmp/lt_hourly_error_data.txt
LIMIT=0
DATE=`date +"%Y%m"`

cat log-[12]-${DATE}*_*.txt log-1.txt log-2.txt | grep "`date +"%Y-%m-%d %H:"`" | \
  egrep "An error has occurred" | \
  grep -v "Could not decode query" | \
  grep -v "URLDecoder: Illegal hex characters" | \
  grep -v "Missing 'language' parameter" | \
  grep -v "Text checking took longer than" | \
  grep -v "Missing 'text' or 'data' parameter" | \
  grep -v "is not a language code known" | \
  grep -v "Unsupported action: 'v2/check" | \
  grep -v "You specified 'preferredVariants' but" | \
  grep -v "Unsupported action: ''" | \
  grep -v "code 413" >$TMPDATA
#cat log-1.txt log-2.txt | grep "`date +"%Y-%m-%d %H:"`" | egrep "An error has occurred" | grep -v "code 413" >$TMPDATA
MATCHES=`cat $TMPDATA | wc -l`

if [ "$MATCHES" -gt "$LIMIT" ]
then
  cat $TMPDATA | tail -n 100 | \
        ifne mail -a 'Content-Type: text/plain; charset=utf-8' -s "Hourly LT Error Report ($MATCHES total)" ${FIRSTPART}@${LASTPART}
fi
{'fraction_non_alphanumeric': 0.1819484240687679, 'fraction_numerical': 0.024832855778414518, 'mean_word_length': 3.3828451882845187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '816816', 'n_tokens_mistral': 900, 'n_tokens_neox': 843, 'n_words': 245}