<commit_before>! Copyright (C) 2011 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: assocs combinators continuations fry kernel lexer locals
math math.functions math.order math.parser sequences splitting ;
IN: units.reduction

CONSTANT: storage-suffixes { "B" "K" "M" "G" "T" "P" "E" "Z" "Y" }

CONSTANT: unit-suffix-hash H{
        { CHAR: B 0 } { CHAR: K 1 } { CHAR: M 2 } { CHAR: G 3 }
        { CHAR: T 4 } { CHAR: P 5 } { CHAR: E 6 } { CHAR: Z 7 }
        { CHAR: Y 8 }
    }

: threshhold ( n multiplier base -- x )
    [ * ] dip swap ^ ; inline

:: find-unit-suffix ( suffixes n multiplier base -- i/f )
    suffixes length
    [ [ n ] dip multiplier base threshhold < ] find-integer
   suffixes length or 1 [-] ;

:: reduce-magnitude ( n multiplier base suffixes -- string )
    n 0 < [
        n neg multiplier base suffixes reduce-magnitude
        "-" prepend
    ] [
        suffixes n multiplier base find-unit-suffix :> i
        n multiplier i * base swap ^
        /i number>string i suffixes nth append
    ] if ;

: n>storage ( n -- string )
    10 2 storage-suffixes reduce-magnitude "i" append ;

: n>Storage ( n -- string )
    3 10 storage-suffixes reduce-magnitude ;

ERROR: bad-storage-string string reason ;

:: (storage>n) ( string multiplier base -- n )
    string last unit-suffix-hash ?at [
        :> unit
        string but-last string>number
        [ "not a number" throw ] unless*
        multiplier unit * base swap ^ *
    ] [
        "unrecognized unit" throw<commit_msg>units.reduction: avoid double length calls in find-unit-suffix.<commit_after>! Copyright (C) 2011 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: assocs combinators continuations fry kernel lexer locals
math math.functions math.order math.parser sequences splitting ;
IN: units.reduction

CONSTANT: storage-suffixes { "B" "K" "M" "G" "T" "P" "E" "Z" "Y" }

CONSTANT: unit-suffix-hash H{
        { CHAR: B 0 } { CHAR: K 1 } { CHAR: M 2 } { CHAR: G 3 }
        { CHAR: T 4 } { CHAR: P 5 } { CHAR: E 6 } { CHAR: Z 7 }
        { CHAR: Y 8 }
    }

: threshhold ( n multiplier base -- x )
    [ * ] dip swap ^ ; inline

:: find-unit-suffix ( suffixes n multiplier base -- i/f )
    suffixes length [
        [ [ n ] dip multiplier base threshhold < ] find-integer
    ] keep or 1 [-] ;

:: reduce-magnitude ( n multiplier base suffixes -- string )
    n 0 < [
        n neg multiplier base suffixes reduce-magnitude
        "-" prepend
    ] [
        suffixes n multiplier base find-unit-suffix :> i
        n multiplier i * base swap ^
        /i number>string i suffixes nth append
    ] if ;

: n>storage ( n -- string )
    10 2 storage-suffixes reduce-magnitude "i" append ;

: n>Storage ( n -- string )
    3 10 storage-suffixes reduce-magnitude ;

ERROR: bad-storage-string string reason ;

:: (storage>n) ( string multiplier base -- n )
    string last unit-suffix-hash ?at [
        :> unit
        string but-last string>number
        [ "not a number" throw ] unless*
        multiplier unit * base swap ^ *
    ] [
        "unrecognized unit" throw
{'fraction_non_alphanumeric': 0.1211439588688946, 'fraction_numerical': 0.013496143958868894, 'mean_word_length': 2.343716433941998, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4724401', 'n_tokens_mistral': 1084, 'n_tokens_neox': 979, 'n_words': 346}