<commit_before>! Copyright (C) 2008, 2010 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors fry kernel make math math.order math.parser
sequences sorting.functor strings unicode.case
unicode.categories unicode.collation ;
IN: sorting.human

: cut-find ( sequence pred -- before after )
    [ drop ] [ find drop ] 2bi dup [ cut ] when ; inline

: cut3 ( sequence pred -- first mid last )
    [ cut-find ] keep [ not ] compose cut-find ; inline

: find-sequences ( sequence pred quot -- sequences )
    '[
        [
            _ cut3 [
                [ , ]
                [ [ @ , ] when* ] bi*
            ] dip dup
        ] loop drop
    ] { } make ; inline

: find-numbers ( sequence -- sequence' )
    [ digit? ] [ string>number ] find-sequences ;

! For comparing integers or sequences
TUPLE: hybrid obj ;

: <hybrid> ( obj -- hybrid )
    hybrid new
        swap >>obj ; inline

: <hybrid-insensitive> ( obj -- hybrid )
    hybrid new
        swap dup string? [ w/collation-key ] when >>obj ; inline

M: hybrid <=>
    [ obj>> ] bi@
    2dup [ integer? ] bi@ xor [
        drop integer? +lt+ +gt+ ?
    ] [
        <=>
    ] if ;

<< "human" [ find-numbers [ <hybrid> ] map ] define-sorting >>
<< "humani" [ find-numbers [ <hybrid-insensitive> ] map ] define-sorting >><commit_msg>sorting.human: rename <hybrid> to <alphanum>.  Fixes #226.<commit_after>! Copyright (C) 2008, 2010 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors fry kernel make math math.order math.parser
sequences sorting.functor strings unicode.case
unicode.categories unicode.collation ;
IN: sorting.human

: cut-find ( sequence pred -- before after )
    [ drop ] [ find drop ] 2bi dup [ cut ] when ; inline

: cut3 ( sequence pred -- first mid last )
    [ cut-find ] keep [ not ] compose cut-find ; inline

: find-sequences ( sequence pred quot -- sequences )
    '[
        [
            _ cut3 [
                [ , ]
                [ [ @ , ] when* ] bi*
            ] dip dup
        ] loop drop
    ] { } make ; inline

: find-numbers ( sequence -- sequence' )
    [ digit? ] [ string>number ] find-sequences ;

! For comparing integers or sequences
TUPLE: alphanum obj ;

: <alphanum> ( obj -- hybrid )
    alphanum new
        swap >>obj ; inline

: <alphanum-insensitive> ( obj -- hybrid )
    alphanum new
        swap dup string? [ w/collation-key ] when >>obj ; inline

M: alphanum <=>
    [ obj>> ] bi@
    2dup [ integer? ] bi@ xor [
        drop integer? +lt+ +gt+ ?
    ] [
        <=>
    ] if ;

<< "human" [ find-numbers [ <alphanum> ] map ] define-sorting >>
<< "humani" [ find-numbers [ <alphanum-insensitive> ] map ] define-sorting >>
{'fraction_non_alphanumeric': 0.1380198748619801, 'fraction_numerical': 0.009937430990062569, 'mean_word_length': 2.2550898203592813, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2992067', 'n_tokens_mistral': 900, 'n_tokens_neox': 843, 'n_words': 254}