<commit_before>! Copyright (C) 2022 CapitalEx
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays assocs compiler.units continuations
formatting hash-sets hashtables io io.encodings.utf8 io.files
kernel namespaces regexp sequences sequences.deep sets sorting
splitting unicode vocabs vocabs.loader ;
FROM: namespaces => set ;
IN: lint.vocabs

<PRIVATE
SYMBOL: old-dictionary

: save-dictionary ( -- )
    dictionary     get clone 
    old-dictionary set       ;

: restore-dictionary ( -- )
    dictionary     get keys >hash-set
    old-dictionary get keys >hash-set
    diff members [ [ forget-vocab ] each ] with-compilation-unit ;

: vocab-loaded? ( name -- ? )
    dictionary get key? ;

: (get-words) ( name -- vocab )
    dup load-vocab words>> keys 2array ;

: no-vocab-found ( name -- empty )
    { } 2array ;

: nl>space ( string -- string )
    "\n" " " replace ;

: find-import-statements ( string -- seq )
    "USING: [^;]+ ;|USE: \\S+" <regexp> all-matching-subseqs ;

: clean-up-source ( string -- string )
    "\"(\\\"|[^\"]*)\"|(R/ (\\\\/|[^/])*/)|\\\\\\s+\\S+|POSTPONE: \\S+|! ([^\n])*" <regexp> "" re-replace ;

: strip-syntax ( seq -- seq )
    [ "USING: | ;|USE: " <regexp> " " re-replace ] map ;

: split-when-blank ( string -- seq )
    [ blank? ] split-when ;

: split-words ( line -- words )
    [ split-when-blank ] map flatten harvest ;

: get-unique-words ( seq -- hash-set )
    harvest split-words >hash-set ;

: [is-used?] ( hash-set  -- quot )
    '[ nip [ _ in? ] any? ] ; inline

: reject-unused-vocabs ( assoc hash-set -- seq )
    [is-used?] assoc-reject keys ;
<commit_msg>Made RegExp More Specific<commit_after>! Copyright (C) 2022 CapitalEx
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays assocs compiler.units continuations
formatting hash-sets hashtables io io.encodings.utf8 io.files
kernel namespaces regexp sequences sequences.deep sets sorting
splitting unicode vocabs vocabs.loader ;
FROM: namespaces => set ;
IN: lint.vocabs

<PRIVATE
SYMBOL: old-dictionary

: save-dictionary ( -- )
    dictionary     get clone 
    old-dictionary set       ;

: restore-dictionary ( -- )
    dictionary     get keys >hash-set
    old-dictionary get keys >hash-set
    diff members [ [ forget-vocab ] each ] with-compilation-unit ;

: vocab-loaded? ( name -- ? )
    dictionary get key? ;

: (get-words) ( name -- vocab )
    dup load-vocab words>> keys 2array ;

: no-vocab-found ( name -- empty )
    { } 2array ;

: nl>space ( string -- string )
    "\n" " " replace ;

: find-import-statements ( string -- seq )
    "USING: [^;]+ ;|USE: \\S+" <regexp> all-matching-subseqs ;

: clean-up-source ( string -- string )
    "\"(\\\"|[^\"]*|\n)\"|(R/ (\\\\/|[^/])*/)|\\\\\\s+(USE:|USING:)|POSTPONE:\\s+(USE:|USING:)|! ([^\n])*" <regexp> "" re-replace ;

: strip-syntax ( seq -- seq )
    [ "USING: | ;|USE: " <regexp> " " re-replace ] map ;

: split-when-blank ( string -- seq )
    [ blank? ] split-when ;

: split-words ( line -- words )
    [ split-when-blank ] map flatten harvest ;

: get-unique-words ( seq -- hash-set )
    harvest split-words >hash-set ;

: [is-used?] ( hash-set  -- quot )
    '[ nip [ _ in? ] any? ] ; inline

: reject-unused-vocabs ( assoc hash-set -- seq )
    [is-used?] assoc-reject keys ;

{'fraction_non_alphanumeric': 0.17904589371980675, 'fraction_numerical': 0.004227053140096618, 'mean_word_length': 2.6648230088495577, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1532370', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1163, 'n_words': 282}