<commit_before>! Copyright (C) 2009 <NAME>, <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: accessors alien.c-types byte-arrays combinators fry
grouping half-floats images kernel math math.vectors sequences
specialized-arrays specialized-arrays.instances.float
specialized-arrays.instances.half
specialized-arrays.instances.uint
specialized-arrays.instances.ushort words ;
FROM: alien.c-types => float ;
IN: images.normalization

<PRIVATE

CONSTANT: don't-care 3

: permutation ( src dst -- seq n )
    [
        swap '[ _ index [ don't-care ] unless* ] { } map-as
        4 don't-care pad-tail
    ] keep length ;

: pad4 ( seq -- newseq ) 4 255 pad-tail ;

: permute ( byte-array src-order dst-order -- byte-array )
   [ name>> [ length ] keep ] [ name>> ] bi*
   permutation [ group ] 2dip '[ pad4 _ vshuffle _ head ] map concat ;

: (reorder-colors) ( image src-order dest-order -- image )
    [ permute ] 2curry change-bitmap ;

GENERIC: normalize-component-type* ( image component-type -- image )

: normalize-floats ( float-array -- byte-array )
    [ 255.0 * >integer ] B{ } map-as ;

M: float-components normalize-component-type*
    drop byte-array>float-array normalize-floats ;

M: half-components normalize-component-type*
    drop byte-array>half-array normalize-floats ;

: ushorts>ubytes ( bitmap -- bitmap' )
    byte-array>ushort-array [ -8 shift ] B{ } map-as ; inline

M: ushort-components normalize-component-type*
    drop ushorts>ubytes ;

M: ubyte-components normalize-component-type*
    drop ;<commit_msg>images.normalization: refactored to use locals<commit_after>! Copyright (C) 2009 <NAME>, <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: accessors alien.c-types byte-arrays combinators fry
grouping half-floats images kernel locals math math.vectors
sequences specialized-arrays specialized-arrays.instances.float
specialized-arrays.instances.half
specialized-arrays.instances.uint
specialized-arrays.instances.ushort words ;
FROM: alien.c-types => float ;
IN: images.normalization

<PRIVATE

CONSTANT: don't-care 3

: permutation ( src dst -- seq )
    swap '[ _ index [ don't-care ] unless* ] { } map-as
    4 don't-care pad-tail ;

: pad4 ( seq -- newseq ) 4 255 pad-tail ;

:: permute ( bytes src-order dst-order -- new-bytes )
    [let | src [ src-order name>> ]
           dst [ dst-order name>> ] |
        bytes src length group
        [ pad4 src dst permutation vshuffle dst length head ]
        map concat ] ;

: (reorder-colors) ( image src-order dest-order -- image )
    [ permute ] 2curry change-bitmap ;

GENERIC: normalize-component-type* ( image component-type -- image )

: normalize-floats ( float-array -- byte-array )
    [ 255.0 * >integer ] B{ } map-as ;

M: float-components normalize-component-type*
    drop byte-array>float-array normalize-floats ;

M: half-components normalize-component-type*
    drop byte-array>half-array normalize-floats ;

: ushorts>ubytes ( bitmap -- bitmap' )
    byte-array>ushort-array [ -8 shift ] B{ } map-as ; inline

M: ushort-components normalize-component-type*
    drop ushorts>ubytes ;

M: ubyte-components normalize-component-type*
    drop ;
{'fraction_non_alphanumeric': 0.12315425699026077, 'fraction_numerical': 0.011624253848570531, 'mean_word_length': 3.3437926330150067, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5928652', 'n_tokens_mistral': 1091, 'n_tokens_neox': 1009, 'n_words': 266}