<commit_before>! Copyright (C) 2008, 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: unicode.data kernel math sequences parser
bit-arrays namespaces sequences.private arrays classes.parser
assocs classes.predicate sets fry splitting accessors ;
IN: unicode.categories.syntax

! For use in CATEGORY:
SYMBOLS: Cn Lu Ll Lt Lm Lo Mn Mc Me Nd Nl No Pc Pd Ps Pe Pi Pf Po Sm Sc Sk So Zs Zl Zp Cc Cf Cs Co | ;

<PRIVATE

: >category-array ( categories -- bitarray )
    categories [ swap member? ] with map >bit-array ;

: [category] ( categories code -- quot )
    [ >category-array ] dip
    '[ dup category# _ nth-unsafe [ drop t ] _ if ] ;

: integer-predicate-class ( word predicate -- )
    integer swap define-predicate-class ;

: define-category ( word categories code -- )
    [category] integer-predicate-class ;

: define-not-category ( word categories code -- )
    [category] [ not ] compose integer-predicate-class ;

: parse-category ( -- word tokens quot )
    CREATE-CLASS \ ; parse-until { | } split1
    [ [ name>> ] map ]
    [ [ [ ] like ] [ [ drop f ] ] if* ] bi* ;

PRIVATE>

: CATEGORY:
    parse-category define-category ; parsing

: CATEGORY-NOT:
    parse-category define-not-category ; parsing<commit_msg>Making unicode.categories.syntax use member?, simplifying the code<commit_after>! Copyright (C) 2008, 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: unicode.data kernel math sequences parser unicode.data.private
bit-arrays namespaces sequences.private arrays classes.parser
assocs classes.predicate sets fry splitting accessors ;
IN: unicode.categories.syntax

! For use in CATEGORY:
SYMBOLS: Cn Lu Ll Lt Lm Lo Mn Mc Me Nd Nl No Pc Pd Ps Pe Pi Pf Po Sm Sc Sk So Zs Zl Zp Cc Cf Cs Co | ;

<PRIVATE

: [category] ( categories code -- quot )
    '[ dup category# _ member? [ drop t ] _ if ] ;

: integer-predicate-class ( word predicate -- )
    integer swap define-predicate-class ;

: define-category ( word categories code -- )
    [category] integer-predicate-class ;

: define-not-category ( word categories code -- )
    [category] [ not ] compose integer-predicate-class ;

: parse-category ( -- word tokens quot )
    CREATE-CLASS \ ; parse-until { | } split1
    [ [ name>> categories-map at ] map ]
    [ [ [ ] like ] [ [ drop f ] ] if* ] bi* ;

PRIVATE>

: CATEGORY:
    parse-category define-category ; parsing

: CATEGORY-NOT:
    parse-category define-not-category ; parsing
{'fraction_non_alphanumeric': 0.11421628189550426, 'fraction_numerical': 0.007290400972053463, 'mean_word_length': 3.1864406779661016, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2484956', 'n_tokens_mistral': 822, 'n_tokens_neox': 764, 'n_words': 257}