<commit_before>import re

BASE_FLAGS = frozenset([
    '@provideGoog'
    ])

JSDOC_FLAGS = frozenset([
    '@suppress'
    ])

FILE_FLAGS = frozenset([
    '@author',
    '@fileoverview',
    '@see',
    '@license',
    '@visibility'
    ])

FUNCTION_FLAGS = frozenset([
    '@param',
    '@return',
    '@template',
    '@deprecated',
    '@throws',
    '@see',
    '@override',
    ])

VISIBILITY_FLAGS = frozenset([
    # Everything public by default
    '@protected',
    '@private'
    ])

INSTANTIABLE_FLAGS = frozenset([
    '@constructor',
    '@extends',
    '@implements',
    '@see',
    ])

TYPEDEF_FLAGS = frozenset([
    '@typedef'
    ])

PROPERTY_FLAGS = frozenset([
    '@const',
    '@define',    
    '@enum',
    '@struct',
    '@type',
    '@inheritDoc'
    ])

INTERFACE_FLAGS = frozenset([
    '@interface',
    '@extends'
    ])

COMPILER_FLAGS = frozenset([
    '@nocompile',
    '@preserveTry',    
    ])

# TODO(nanaze): File.
MISC_FLAGS = frozenset([
    '@desc', 
    '@supported',
    '@hidden',
    '@final',
    '@this'    
    ])

all_flags = set()
all_flags.update(MISC_FLAGS)
all_flags.update(BASE_FLAGS)
all_flags.update(COMPILER_FLAGS)
all_flags.update(JSDOC_FLAGS)
all_flags.update(FILE_FLAGS)
all_flags.update(FUNCTION_FLAGS)
all_flags.update(INTERFACE_FLAGS)
all_flags.update(INSTANTIABLE_FLAGS)
all_flags.update(PROPERTY_FLAGS)
all_flags.update(TYPEDEF_FLAGS)
all_flags.update(VISIBILITY_FLAGS)

ALL_FLAGS = frozenset(all_flags)


def ParseParameterDescription(desc):
  match = re.match(r'^\s*\{(?P<type>.*?)\}\s+(?P<name>\w+)(?P<desc>.*)$', desc, re.DOTALL | re.MULTILINE)
  if not match:
    raise ValueError('Could not parse flag description: %s' % desc)<commit_msg>Add a new tag for idGenerator<commit_after>import re

BASE_FLAGS = frozenset([
    '@provideGoog'
    ])

JSDOC_FLAGS = frozenset([
    '@suppress'
    ])

FILE_FLAGS = frozenset([
    '@author',
    '@fileoverview',
    '@see',
    '@license',
    '@visibility'
    ])

FUNCTION_FLAGS = frozenset([
    '@param',
    '@return',
    '@template',
    '@deprecated',
    '@throws',
    '@see',
    '@override',
    ])

VISIBILITY_FLAGS = frozenset([
    # Everything public by default
    '@protected',
    '@private'
    ])

INSTANTIABLE_FLAGS = frozenset([
    '@constructor',
    '@extends',
    '@implements',
    '@see',
    ])

TYPEDEF_FLAGS = frozenset([
    '@typedef'
    ])

PROPERTY_FLAGS = frozenset([
    '@const',
    '@define',    
    '@enum',
    '@struct',
    '@type',
    '@inheritDoc'
    ])

INTERFACE_FLAGS = frozenset([
    '@interface',
    '@extends'
    ])

COMPILER_FLAGS = frozenset([
    '@nocompile',
    '@preserveTry',    
    ])

# TODO(nanaze): File.
MISC_FLAGS = frozenset([
    '@desc', 
    '@supported',
    '@hidden',
    '@final',
    '@idGenerator',
    '@this'    
    ])

all_flags = set()
all_flags.update(MISC_FLAGS)
all_flags.update(BASE_FLAGS)
all_flags.update(COMPILER_FLAGS)
all_flags.update(JSDOC_FLAGS)
all_flags.update(FILE_FLAGS)
all_flags.update(FUNCTION_FLAGS)
all_flags.update(INTERFACE_FLAGS)
all_flags.update(INSTANTIABLE_FLAGS)
all_flags.update(PROPERTY_FLAGS)
all_flags.update(TYPEDEF_FLAGS)
all_flags.update(VISIBILITY_FLAGS)

ALL_FLAGS = frozenset(all_flags)


def ParseParameterDescription(desc):
  match = re.match(r'^\s*\{(?P<type>.*?)\}\s+(?P<name>\w+)(?P<desc>.*)$', desc, re.DOTALL | re.MULTILINE)
  if not match:
    raise ValueError('Could not parse flag description: %s' % desc)
{'fraction_non_alphanumeric': 0.17052023121387283, 'fraction_numerical': 0.0, 'mean_word_length': 3.5241830065359476, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3445072', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1330, 'n_words': 203}