<commit_before>"""Module to analyze Python source code; for syntax coloring tools.

Interface:
    for tag, start, end, sublist in fontify(pytext, searchfrom, searchto):
        ...

The 'pytext' argument is a string containing Python source code.
The (optional) arguments 'searchfrom' and 'searchto' may contain a slice in pytext. 
The returned value is a list of tuples, formatted like this:
    [('keyword', 0, 6, None), ('keyword', 11, 17, None), ('comment', 23, 53, None), etc. ]
The tuple contents are always like this:
    (tag, startindex, endindex, sublist)
tag is one of 'keyword', 'string', 'comment' or 'identifier'
sublist is not used, hence always None. 
"""

# Based on FontText.py by <NAME>,
# which was modified by <NAME>,
# then un-Tk'd by <NAME>ssum.
# Many thanks for regular expression debugging & authoring are due to:
#	Tim (the-incredib-ly y'rs) Peters and <NAME>
# So, who owns the copyright? ;-) How about this:
# Copyright 1996-2003:
#	<NAME>,
#	<NAME>,
#	<NAME>,
#	<NAME>

# from __future__ import generators

__version__ = "0.5"

import re
import graphics
import util

from keyword import kwlist as keywordsList
keywordsList = keywordsList[:]
keywordsList += ["None", "True", "False"]
keywordsList += graphics.__all__
keywordsList += util.__all__
keywordsList += dir(graphics.Context)

# Build up a regular expression which will match anything<commit_msg>Added missing keywords for syntax coloring: MOUSEX, MOUSEY, mousedown, keydown, key, scrollwheel, wheeldelta.<commit_after>"""Module to analyze Python source code; for syntax coloring tools.

Interface:
    for tag, start, end, sublist in fontify(pytext, searchfrom, searchto):
        ...

The 'pytext' argument is a string containing Python source code.
The (optional) arguments 'searchfrom' and 'searchto' may contain a slice in pytext. 
The returned value is a list of tuples, formatted like this:
    [('keyword', 0, 6, None), ('keyword', 11, 17, None), ('comment', 23, 53, None), etc. ]
The tuple contents are always like this:
    (tag, startindex, endindex, sublist)
tag is one of 'keyword', 'string', 'comment' or 'identifier'
sublist is not used, hence always None. 
"""

# Based on FontText.py by <NAME>,
# which was modified by <NAME>,
# then un-Tk'd by Just van Rossum.
# Many thanks for regular expression debugging & authoring are due to:
#	Tim (the-incredib-ly y'rs) Peters and <NAME>
# So, who owns the copyright? ;-) How about this:
# Copyright 1996-2003:
#	<NAME>,
#	<NAME>,
#	<NAME>,
#	<NAME>

# from __future__ import generators

__version__ = "0.5"

import re
import graphics
import util

from keyword import kwlist as keywordsList
keywordsList = keywordsList[:]
keywordsList += ["None", "True", "False"]
keywordsList += graphics.__all__
keywordsList += util.__all__
keywordsList += dir(graphics.Context)
keywordsList += ["MOUSEX", "MOUSEY", "mousedown", "keydown", "key", "scrollwheel", "wheeldelta"]


# Build up a regular expression which will match anything
{'fraction_non_alphanumeric': 0.12066059993259184, 'fraction_numerical': 0.013481631277384564, 'mean_word_length': 4.685823754789272, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7180161', 'n_tokens_mistral': 947, 'n_tokens_neox': 893, 'n_words': 373}