<commit_before>USING: accessors ascii assocs biassocs lexer sequences ;

IN: unicode.flags

MEMO: flag-codes ( -- biassoc ) H{
    { CHAR: A CHAR: 🇦 }
    { CHAR: B CHAR: 🇧 }
    { CHAR: C CHAR: 🇨 }
    { CHAR: D CHAR: 🇩 }
    { CHAR: E CHAR: 🇪 }
    { CHAR: F CHAR: 🇫 }
    { CHAR: G CHAR: 🇬 }
    { CHAR: H CHAR: 🇭 }
    { CHAR: I CHAR: 🇮 }
    { CHAR: J CHAR: 🇯 }
    { CHAR: K CHAR: 🇰 }
    { CHAR: L CHAR: 🇱 }
    { CHAR: M CHAR: 🇲 }
    { CHAR: N CHAR: 🇳 }
    { CHAR: O CHAR: 🇴 }
    { CHAR: P CHAR: 🇵 }
    { CHAR: Q CHAR: 🇶 }
    { CHAR: R CHAR: 🇷 }
    { CHAR: S CHAR: 🇸 }
    { CHAR: T CHAR: 🇹 }
    { CHAR: U CHAR: 🇺 }
    { CHAR: V CHAR: 🇻 }
    { CHAR: W CHAR: 🇼 }
    { CHAR: X CHAR: 🇽 }
    { CHAR: Y CHAR: 🇾 }
    { CHAR: Z CHAR: 🇿 }
} >biassoc ;

: unicode>flag ( country-code -- flag )
    >upper [ flag-codes from>> at ] map ;

: flag>unicode ( flag -- country-code )
    [ flag-codes to>> at ] map ;

SYNTAX: FLAG: scan-token unicode>flag suffix! ;<commit_msg>unicode.flags: add some more flags and a way to view the codepoints for fun<commit_after>USING: accessors assocs biassocs kernel lexer prettyprint
sequences unicode ;

IN: unicode.flags

MEMO: flag-codes ( -- biassoc ) H{
    { CHAR: A CHAR: 🇦 }
    { CHAR: B CHAR: 🇧 }
    { CHAR: C CHAR: 🇨 }
    { CHAR: D CHAR: 🇩 }
    { CHAR: E CHAR: 🇪 }
    { CHAR: F CHAR: 🇫 }
    { CHAR: G CHAR: 🇬 }
    { CHAR: H CHAR: 🇭 }
    { CHAR: I CHAR: 🇮 }
    { CHAR: J CHAR: 🇯 }
    { CHAR: K CHAR: 🇰 }
    { CHAR: L CHAR: 🇱 }
    { CHAR: M CHAR: 🇲 }
    { CHAR: N CHAR: 🇳 }
    { CHAR: O CHAR: 🇴 }
    { CHAR: P CHAR: 🇵 }
    { CHAR: Q CHAR: 🇶 }
    { CHAR: R CHAR: 🇷 }
    { CHAR: S CHAR: 🇸 }
    { CHAR: T CHAR: 🇹 }
    { CHAR: U CHAR: 🇺 }
    { CHAR: V CHAR: 🇻 }
    { CHAR: W CHAR: 🇼 }
    { CHAR: X CHAR: 🇽 }
    { CHAR: Y CHAR: 🇾 }
    { CHAR: Z CHAR: 🇿 }
} >biassoc ;

: unicode>flag ( country-code -- flag )
    >upper [ flag-codes from>> at ] map ;

: flag>unicode ( flag -- country-code )
    [ flag-codes to>> at ] map ;

! Random flags, England/Scotland/Wales, Refugee Nation Flag
CONSTANT: extra-flags { "🏁" "🚩" "🎌" "🏴" "🏳" "🏳️‍🌈" "🏴‍☠️" "🏴󠁧󠁢󠁥󠁮󠁧󠁿" "🏴󠁧󠁢󠁳󠁣󠁴󠁿" "🏴󠁧󠁢󠁷󠁬󠁳󠁿" "🏳️‍🟧‍⬛️‍🟧" }

: explain-extra-flags ( -- )
    extra-flags [
        dup . [ dup char>name ] { } map>assoc .
    ] each ;


SYNTAX: FLAG: scan-token unicode>flag suffix! ;
{'fraction_non_alphanumeric': 0.19628990509059535, 'fraction_numerical': 0.0, 'mean_word_length': 2.01953125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3691359', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1122, 'n_words': 317}