<commit_before>#ifndef __IMAP_PARSER_H
#define __IMAP_PARSER_H

/* FIXME: we don't have ']' here due to FETCH BODY[] handling failing
   with it.. */
#define IS_ATOM_SPECIAL(c) \
	((c) == '(' || (c) == ')' || (c) == '{' || (c) == '%' || (c) == '*' || \
	 (c) == '"' || (c) == '\\' || (c) <= 32 || (c) == 0x7f)

enum imap_parser_flags {
	/* Set this flag if you wish to read only size of literal argument
	   and not convert literal into string. Useful when you need to deal
	   with large literal sizes. The literal must be the last read<commit_msg>And removed the other added atom-specials as well..<commit_after>#ifndef __IMAP_PARSER_H
#define __IMAP_PARSER_H

/* FIXME: we don't have ']' here due to FETCH BODY[] handling failing
   with it.. also '%' and '*' are banned due to LIST, and '\' due to it being
   in flags. oh well.. */
#define IS_ATOM_SPECIAL(c) \
	((c) == '(' || (c) == ')' || (c) == '{' || \
	 (c) == '"' || (c) <= 32 || (c) == 0x7f)

enum imap_parser_flags {
	/* Set this flag if you wish to read only size of literal argument
	   and not convert literal into string. Useful when you need to deal
	   with large literal sizes. The literal must be the last read
{'fraction_non_alphanumeric': 0.165961049957663, 'fraction_numerical': 0.006773920406435224, 'mean_word_length': 3.2981818181818183, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6481490', 'n_tokens_mistral': 430, 'n_tokens_neox': 415, 'n_words': 155}