<commit_before>/* attribute identifiers */
DEF(TOK_SECTION, "section")
DEF(TOK___SECTION__, "__section__")
DEF(TOK_ALIGNED, "aligned")
DEF(TOK___ALIGNED__, "__aligned__")
DEF(TOK_UNUSED, "unused")
DEF(TOK___UNUSED__, "__unused__")
DEF(TOK_CDECL, "cdecl")
DEF(TOK___CDECL, "__cdecl")
DEF(TOK___CDECL__, "__cdecl__")
DEF(TOK_STDCALL, "stdcall")
DEF(TOK___STDCALL, "__stdcall")
DEF(TOK___STDCALL__, "__stdcall__")
DEF(TOK_NORETURN, "noreturn")
DEF(TOK___NORETURN__, "__noreturn__")

/* builtin functions */
DEF(TOK_memcpy, "memcpy")
DEF(TOK_memset, "memset")
DEF(TOK___divdi3, "__divdi3")
DEF(TOK___moddi3, "__moddi3")
DEF(TOK___udivdi3, "__udivdi3")
DEF(TOK___umoddi3, "__umoddi3")
DEF(TOK___sardi3, "__sardi3")
DEF(TOK___shrdi3, "__shrdi3")
DEF(TOK___shldi3, "__shldi3")
DEF(TOK___ulltof, "__ulltof")
DEF(TOK___ulltod, "__ulltod")
DEF(TOK___ulltold, "__ulltold")
DEF(TOK___fixunssfdi, "__fixunssfdi")
DEF(TOK___fixunsdfdi, "__fixunsdfdi")
DEF(TOK___fixunsxfdi, "__fixunsxfdi")
DEF(TOK___bound_ptr_add, "__bound_ptr_add")
DEF(TOK___bound_ptr_indir1, "__bound_ptr_indir1")
DEF(TOK___bound_ptr_indir2, "__bound_ptr_indir2")
DEF(TOK___bound_ptr_indir4, "__bound_ptr_indir4")
DEF(TOK___bound_ptr_indir8, "__bound_ptr_indir8")
DEF(TOK___bound_ptr_indir12, "__bound_ptr_indir12")
DEF(TOK___bound_ptr_indir16, "__bound_ptr_indir16")<commit_msg>added bound check symbols<commit_after>/* attribute identifiers */
DEF(TOK_SECTION, "section")
DEF(TOK___SECTION__, "__section__")
DEF(TOK_ALIGNED, "aligned")
DEF(TOK___ALIGNED__, "__aligned__")
DEF(TOK_UNUSED, "unused")
DEF(TOK___UNUSED__, "__unused__")
DEF(TOK_CDECL, "cdecl")
DEF(TOK___CDECL, "__cdecl")
DEF(TOK___CDECL__, "__cdecl__")
DEF(TOK_STDCALL, "stdcall")
DEF(TOK___STDCALL, "__stdcall")
DEF(TOK___STDCALL__, "__stdcall__")
DEF(TOK_NORETURN, "noreturn")
DEF(TOK___NORETURN__, "__noreturn__")

/* builtin functions or variables */
DEF(TOK_memcpy, "memcpy")
DEF(TOK_memset, "memset")
DEF(TOK___divdi3, "__divdi3")
DEF(TOK___moddi3, "__moddi3")
DEF(TOK___udivdi3, "__udivdi3")
DEF(TOK___umoddi3, "__umoddi3")
DEF(TOK___sardi3, "__sardi3")
DEF(TOK___shrdi3, "__shrdi3")
DEF(TOK___shldi3, "__shldi3")
DEF(TOK___tcc_int_fpu_control, "__tcc_int_fpu_control")
DEF(TOK___tcc_fpu_control, "__tcc_fpu_control")
DEF(TOK___ulltof, "__ulltof")
DEF(TOK___ulltod, "__ulltod")
DEF(TOK___ulltold, "__ulltold")
DEF(TOK___fixunssfdi, "__fixunssfdi")
DEF(TOK___fixunsdfdi, "__fixunsdfdi")
DEF(TOK___fixunsxfdi, "__fixunsxfdi")

/* bound checking symbols */
#ifdef CONFIG_TCC_BCHECK
DEF(TOK___bound_ptr_add, "__bound_ptr_add")
DEF(TOK___bound_ptr_indir1, "__bound_ptr_indir1")
DEF(TOK___bound_ptr_indir2, "__bound_ptr_indir2")
DEF(TOK___bound_ptr_indir4, "__bound_ptr_indir4")
DEF(TOK___bound_ptr_indir8, "__bound_ptr_indir8")
DEF(TOK___bound_ptr_indir12, "__bound_ptr_indir12")
DEF(TOK___bound_ptr_indir16, "__bound_ptr_indir16")
DEF(TOK___bound_local_new, "__bound_local_new")
DEF(TOK___bound_local_delete, "__bound_local_delete")
DEF(TOK_malloc, "malloc")
DEF(TOK_free, "free")
DEF(TOK_realloc, "realloc")
DEF(TOK_memalign, "memalign")
DEF(TOK_calloc, "calloc")
DEF(TOK_memmove, "memmove")
DEF(TOK_strlen, "strlen")
DEF(TOK_strcpy, "strlen")
#endif
{'fraction_non_alphanumeric': 0.1411023622047244, 'fraction_numerical': 0.01889763779527559, 'mean_word_length': 14.196172248803828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7067958', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1469, 'n_words': 188}