<commit_before>import itertools

from thinglang.lexer.values.identifier import Identifier

"""
The internal ordering of core types used by the compiler and runtime
"""

INTERNAL_TYPE_COUNTER = itertools.count(1)

# TODO: map dynamically at runtime

INTERNAL_TYPE_ORDERING = {
    Identifier("text"): next(INTERNAL_TYPE_COUNTER),
    Identifier("number"): next(INTERNAL_TYPE_COUNTER),
    Identifier("bool"): next(INTERNAL_TYPE_COUNTER),
    Identifier("list"): next(INTERNAL_TYPE_COUNTER),
    Identifier("map"): next(INTERNAL_TYPE_COUNTER),
    Identifier("iterator"): next(INTERNAL_TYPE_COUNTER),
    Identifier("Console"): next(INTERNAL_TYPE_COUNTER),
    Identifier("File"): next(INTERNAL_TYPE_COUNTER),
    Identifier("Directory"): next(INTERNAL_TYPE_COUNTER),
    Identifier("Time"): next(INTERNAL_TYPE_COUNTER),
    Identifier("Exception"): next(INTERNAL_TYPE_COUNTER)
}
<commit_msg>Remove manual INTERNAL_TYPE_ORDERING map in favor of explicit import tables<commit_after>import glob

import os

from thinglang.lexer.values.identifier import Identifier

"""
The internal ordering of core types used by the compiler and runtime
"""

CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
SOURCE_PATTERN = os.path.join(CURRENT_PATH,  'source/**/*.thing')


def list_types():
    for path in glob.glob(SOURCE_PATTERN, recursive=True):
        name = os.path.basename(path).replace('.thing', '')
        yield name, path


PRIMITIVE_TYPES = [
    'text',
    'number'
]

INTERNAL_SOURCES = {Identifier(name): path for name, path in list_types()}
{'fraction_non_alphanumeric': 0.11863313990973566, 'fraction_numerical': 0.0006447453255963894, 'mean_word_length': 5.747826086956522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3813622', 'n_tokens_mistral': 496, 'n_tokens_neox': 491, 'n_words': 105}