<commit_before>REBOL [
    System: "REBOL [R3] Language Interpreter and Run-time Environment"
    Title: "Low-level strings"
    Rights: {
        Copyright 2012 REBOL Technologies
        REBOL is a trademark of REBOL Technologies
    }
    License: {
        Licensed under the Apache License, Version 2.0.
        See: http://www.apache.org/licenses/LICENSE-2.0
    }
    Purpose: {
        This section holds lower-level C strings used in various parts
        of the system. This section is unique, because it constructs a
        single C string that contains all the specified strings below.
        This is done to eliminate series headers required for normal
        REBOL strings. The strings are referenced using Boot_Strs[RS_*]
        where * is the set-word (and is zero based). For example:
        RS_SCAN+3 refers to "end-of-paren"
    }
]

scan: ; Used by scanner. Keep in sync with Value_Types in scan.h file!
    "end-of-script"
    "newline"
    "block-end"
    "group-end"
    "word"
    "word-set"
    "word-get"
    "word-lit"
    "blank"
    "logic"
    "integer"
    "decimal"
    "percent"
    "money"
    "time"
    "date"
    "char"
    "block-begin"
    "group-begin"
    "string"
    "binary"
    "pair"
    "tuple"
    "file"
    "email"
    "url"
    "issue"
    "tag"
    "path"
    "refine"
    "construct"

info:
    "Booting..."<commit_msg>Synchronize string token table with lexer errors<commit_after>REBOL [
    System: "REBOL [R3] Language Interpreter and Run-time Environment"
    Title: "Low-level strings"
    Rights: {
        Copyright 2012 REBOL Technologies
        REBOL is a trademark of REBOL Technologies
    }
    License: {
        Licensed under the Apache License, Version 2.0.
        See: http://www.apache.org/licenses/LICENSE-2.0
    }
    Purpose: {
        This section holds lower-level C strings used in various parts
        of the system. This section is unique, because it constructs a
        single C string that contains all the specified strings below.
        This is done to eliminate series headers required for normal
        REBOL strings. The strings are referenced using Boot_Strs[RS_*]
        where * is the set-word (and is zero based). For example:
        RS_SCAN+3 refers to "end-of-paren"
    }
]

scan: ; Used by scanner. Keep in sync with Value_Types in scan.h file!
    "end-of-script"
    "newline"
    "block-end"
    "group-end"
    "word"
    "!!!word + 1!!!" ; !!! needed for REB_WORD + (token - TOKEN_WORD) trick
    "!!!word + 2!!!" ; !!! review the necessity of this "adding" on REB_XXXs
    "!!!word + 2!!!" ; !!! see KIND_OF_WORD_FROM_TOKEN
    "set"
    "!!!set + 1!!!"
    "!!!set + 2!!!"
    "!!!set + 3!!!"
    "get"
    "!!!get + 1!!!"
    "!!!get + 2!!!"
    "!!!get + 3!!!"
    "lit"
    "!!!none!!!"     ; not needed
    "bar"
    "lit-bar"
    "blank"
    "!!!logic!!!"    ; not needed
    "integer"
    "decimal"
    "percent"
    "money"
    "time"
    "date"
    "char"
    "block-begin"
    "group-begin"
    "string"
    "binary"
    "pair"
    "tuple"
    "file"
    "email"
    "url"
    "issue"
    "tag"
    "path"
    "refine"
    "construct"

info:
    "Booting..."
{'fraction_non_alphanumeric': 0.12350722815839095, 'fraction_numerical': 0.009113764927718416, 'mean_word_length': 2.205438066465257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1503468', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1051, 'n_words': 332}