<commit_before>#pragma once

#include "axlib.hpp"

#define AX_DICT_ALPHALOWER "a-z"
#define AX_DICT_ALPHAUPPER "A-Z"
#define AX_DICT_DIGITS     "0-9"
#define AX_DICT_MISC       "_"
#define AX_DICT_UNICODE    "+U"

#define AX_DICT_ALPHA      AX_DICT_ALPHALOWER AX_DICT_ALPHAUPPER
#define AX_DICT_ALNUM      AX_DICT_ALPHA      AX_DICT_DIGITS

#define AX_DICT_IDENT      AX_DICT_ALNUM      AX_DICT_MISC

typedef ax_uptr_t          axdict_size_t;
#ifndef axdict_alloc
# define axdict_alloc(N_)  malloc(N_)
#endif
#ifndef axdict_free
# define axdict_free(P_)   free(P_)
#endif

namespace ax
{

	namespace policy
	{

		template< typename TElement >
		struct DictionaryAllocator
		{
			typedef axdict_size_t AllocSizeType;<commit_msg>[ax_dictionary] Don't include the entire axlib with this.<commit_after>#pragma once

#include "ax_platform.h"
#include "ax_types.h"

#include "ax_assert.h"

#define AX_DICT_ALPHALOWER "a-z"
#define AX_DICT_ALPHAUPPER "A-Z"
#define AX_DICT_DIGITS     "0-9"
#define AX_DICT_MISC       "_"
#define AX_DICT_UNICODE    "+U"

#define AX_DICT_ALPHA      AX_DICT_ALPHALOWER AX_DICT_ALPHAUPPER
#define AX_DICT_ALNUM      AX_DICT_ALPHA      AX_DICT_DIGITS

#define AX_DICT_IDENT      AX_DICT_ALNUM      AX_DICT_MISC

typedef ax_uptr_t          axdict_size_t;
#ifndef axdict_alloc
# define axdict_alloc(N_)  malloc(N_)
#endif
#ifndef axdict_free
# define axdict_free(P_)   free(P_)
#endif

namespace ax
{

	namespace policy
	{

		template< typename TElement >
		struct DictionaryAllocator
		{
			typedef axdict_size_t AllocSizeType;
{'fraction_non_alphanumeric': 0.07364341085271318, 'fraction_numerical': 0.002583979328165375, 'mean_word_length': 4.378472222222222, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5752222', 'n_tokens_mistral': 700, 'n_tokens_neox': 623, 'n_words': 124}