<commit_before>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel sequences namespaces words assocs logging sorting
prettyprint io io.styles io.files io.encodings.utf8
strings combinators
logging.server logging.parser calendar.format ;
IN: logging.analysis

SYMBOL: word-names
SYMBOL: errors
SYMBOL: word-histogram
SYMBOL: message-histogram

: analyze-entry ( entry -- )
    dup second ERROR eq? [ dup errors get push ] when
    dup second CRITICAL eq? [ dup errors get push ] when
    1 over third word-histogram get at+
    dup third word-names get member? [
        1 over rest message-histogram get at+
    ] when
    drop ;

: analyze-entries ( entries word-names -- errors word-histogram message-histogram )
    [
        word-names set
        V{ } clone errors set
        H{ } clone word-histogram set
        H{ } clone message-histogram set

        [
            analyze-entry
        ] each

        errors get
        word-histogram get
        message-histogram get
    ] with-scope ;

: histogram. ( assoc quot -- )
    standard-table-style [
        >r >alist sort-values <reversed> r> [
            [ >r swap r> with-cell pprint-cell ] with-row
        ] curry assoc-each
    ] tabular-output ;

: log-entry. ( entry -- )
    "====== " write
    {
        [ first (timestamp>string) bl ]
        [ second pprint bl ]
        [ third write nl ]
        [ fourth "\n" join print ]
    } cleave ;

: errors. ( errors -- )
    [ log-entry. ] each ;

: analysis. ( errors word-histogram message-histogram -- )
    "==== INTERESTING MESSAGES:" print nl
    "Total: " write dup values sum . nl
    [
        dup second write ": " write third "\n" join write
    ] histogram.
    nl
    "==== WORDS:" print nl
    [ write ] histogram.
    nl
    "==== ERRORS:" print nl
    errors. ;

: analyze-log ( lines word-names -- )
    >r parse-log r> analyze-entries analysis. ;

: analyze-log-file ( service word-names -- )
    >r parse-log-file r> analyze-entries analysis. ;
<commit_msg>Fix analysis for recent change<commit_after>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel sequences namespaces words assocs logging sorting
prettyprint io io.styles io.files io.encodings.utf8
strings combinators accessors arrays
logging.server logging.parser calendar.format ;
IN: logging.analysis

SYMBOL: word-names
SYMBOL: errors
SYMBOL: word-histogram
SYMBOL: message-histogram

: analyze-entry ( entry -- )
    dup level>> { ERROR CRITICAL } memq? [ dup errors get push ] when
    1 over word-name>> word-histogram get at+
    dup word-name>> word-names get member? [
        1 over [ level>> ] [ word-name>> ] [ message>> ] tri 3array
        message-histogram get at+
    ] when
    drop ;

: analyze-entries ( entries word-names -- errors word-histogram message-histogram )
    [
        word-names set
        V{ } clone errors set
        H{ } clone word-histogram set
        H{ } clone message-histogram set

        [
            analyze-entry
        ] each

        errors get
        word-histogram get
        message-histogram get
    ] with-scope ;

: histogram. ( assoc quot -- )
    standard-table-style [
        >r >alist sort-values <reversed> r> [
            [ >r swap r> with-cell pprint-cell ] with-row
        ] curry assoc-each
    ] tabular-output ;

: log-entry. ( entry -- )
    "====== " write
    {
        [ date>> (timestamp>string) bl ]
        [ level>> pprint bl ]
        [ word-name>> write nl ]
        [ message>> "\n" join print ]
    } cleave ;

: errors. ( errors -- )
    [ log-entry. ] each ;

: analysis. ( errors word-histogram message-histogram -- )
    "==== INTERESTING MESSAGES:" print nl
    "Total: " write dup values sum . nl
    [
        dup level>> write ": " write message>> "\n" join write
    ] histogram.
    nl
    "==== WORDS:" print nl
    [ write ] histogram.
    nl
    "==== ERRORS:" print nl
    errors. ;

: analyze-log ( lines word-names -- )
    >r parse-log r> analyze-entries analysis. ;

: analyze-log-file ( service word-names -- )
    >r parse-log-file r> analyze-entries analysis. ;

{'fraction_non_alphanumeric': 0.11394755163611046, 'fraction_numerical': 0.0034810860988628454, 'mean_word_length': 1.9785763648928818, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2477507', 'n_tokens_mistral': 1406, 'n_tokens_neox': 1273, 'n_words': 392}