<commit_before>#!/usr/bin/env python

# The following constant specifies the name used in the usage
# statement displayed for python versions lower than 2.3.  (With
# python2.3 and higher the usage statement is generated by optparse
# and uses the actual name of the executable called.)

EXECUTABLE_NAME_FOR_USAGE = "python markdown.py"

SPEED_TEST = 0

"""
====================================================================
IF YOU ARE LOOKING TO EXTEND MARKDOWN, SEE THE "FOOTNOTES" SECTION
====================================================================

Python-Markdown
===============

Converts Markdown to HTML.  Basic usage as a module:

    import markdown
    html = markdown.markdown(your_text_string)

Started by [<NAME>](http://www.dwerg.net/).  Continued and
maintained  by [<NAME>](http://www.freewisdom.org).

Project website: http://www.freewisdom.org/projects/python-markdown
Contact: yuri [at] freewisdom.org

License: GPL 2 (http://www.gnu.org/copyleft/gpl.html) or BSD

Version: 1.6a (October 12, 2006)


For changelog, see the end of file
"""

import re, sys, os, random, codecs

# Set debug level: 3 none, 2 critical, 1 informative, 0 all
(VERBOSE, INFO, CRITICAL, NONE) = range(4)

MESSAGE_THRESHOLD = CRITICAL

def message(level, text) :
    if level >= MESSAGE_THRESHOLD :
        print text


# --------------- CONSTANTS YOU MIGHT WANT TO MODIFY -----------------

TAB_LENGTH = 4            # expand tabs to this many spaces
ENABLE_ATTRIBUTES = True  # @id = xyz -> <... id="xyz">
SMART_EMPHASIS = 1        # this_or_that does not become this<i>or</i>that
HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode

RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'),
                    # from Hebrew to Nko (includes Arabic, Syriac and Thaana)
                    (u'\u2D30', u'\u2D7F'),
                    # Tifinagh
                    )

# Unicode Reference Table:
# 0590-05FF - Hebrew
# 0600-06FF - Arabic
# 0700-074F - Syriac
# 0750-077F - Arabic Supplement
# 0780-07BF - Thaana
# 07C0-07FF - Nko

BOMS = { 'utf-8' : (unicode(codecs.BOM_UTF8, "utf-8"), ),
         'utf-16' : (unicode(codecs.BOM_UTF16_LE, "utf-16"),
                     unicode(codecs.BOM_UTF16_BE, "utf-16")),
         #'utf-32' : (unicode(codecs.BOM_UTF32_LE, "utf-32"),
         #            unicode(codecs.BOM_UTF32_BE, "utf-32")),
         }

def removeBOM(text, encoding):
    for bom in BOMS[encoding]:
        if text.startswith(bom):
            return text.lstrip(bom)
    return text
                    

# --------------- CONSTANTS YOU _SHOULD NOT_ HAVE TO CHANGE ----------

# a template for html placeholders
HTML_PLACEHOLDER_PREFIX = "qaodmasdkwaspemas"
HTML_PLACEHOLDER = HTML_PLACEHOLDER_PREFIX + "%dajkqlsmdqpakldnzsdfls"

BLOCK_LEVEL_ELEMENTS = ['p', 'div', 'blockquote', 'pre', 'table',
                        'dl', 'ol', 'ul', 'script', 'noscript',
                        'form', 'fieldset', 'iframe', 'math', 'ins',
                        'del', 'hr', 'hr/', 'style']

def is_block_level (tag) :
    return ( (tag in BLOCK_LEVEL_ELEMENTS) or
             (tag[0] == 'h' and tag[1] in "0123456789") )

"""
======================================================================
========================== NANODOM ===================================
======================================================================
<commit_msg>Revised the documentation section of the file.  Added version attribute to the module.<commit_after>#!/usr/bin/env python

version = "1.6b"
version_info = (1,6,2,"rc-2")
__revision__ = "$Rev$"

"""
Python-Markdown
===============

Converts Markdown to HTML.  Basic usage as a module:

    import markdown
    md = Markdown()
    html = markdown.convert(your_text_string)

See http://www.freewisdom.org/projects/python-markdown/ for more
information and instructions on how to extend the functionality of the
script.  (You might want to read that before you try modifying this
file.)

Started by [<NAME>](http://www.dwerg.net/).  Continued and
maintained  by [<NAME>](http://www.freewisdom.org).

Contact: yuri [at] freewisdom.org

License: GPL 2 (http://www.gnu.org/copyleft/gpl.html) or BSD

"""


import re, sys, os, random, codecs

# Set debug level: 3 none, 2 critical, 1 informative, 0 all
(VERBOSE, INFO, CRITICAL, NONE) = range(4)

MESSAGE_THRESHOLD = CRITICAL

def message(level, text) :
    if level >= MESSAGE_THRESHOLD :
        print text


# --------------- CONSTANTS YOU MIGHT WANT TO MODIFY -----------------

TAB_LENGTH = 4            # expand tabs to this many spaces
ENABLE_ATTRIBUTES = True  # @id = xyz -> <... id="xyz">
SMART_EMPHASIS = 1        # this_or_that does not become this<i>or</i>that
HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode

RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'),
                    # from Hebrew to Nko (includes Arabic, Syriac and Thaana)
                    (u'\u2D30', u'\u2D7F'),
                    # Tifinagh
                    )

# Unicode Reference Table:
# 0590-05FF - Hebrew
# 0600-06FF - Arabic
# 0700-074F - Syriac
# 0750-077F - Arabic Supplement
# 0780-07BF - Thaana
# 07C0-07FF - Nko

BOMS = { 'utf-8' : (unicode(codecs.BOM_UTF8, "utf-8"), ),
         'utf-16' : (unicode(codecs.BOM_UTF16_LE, "utf-16"),
                     unicode(codecs.BOM_UTF16_BE, "utf-16")),
         #'utf-32' : (unicode(codecs.BOM_UTF32_LE, "utf-32"),
         #            unicode(codecs.BOM_UTF32_BE, "utf-32")),
         }

def removeBOM(text, encoding):
    for bom in BOMS[encoding]:
        if text.startswith(bom):
            return text.lstrip(bom)
    return text

# The following constant specifies the name used in the usage
# statement displayed for python versions lower than 2.3.  (With
# python2.3 and higher the usage statement is generated by optparse
# and uses the actual name of the executable called.)

EXECUTABLE_NAME_FOR_USAGE = "python markdown.py"
                    

# --------------- CONSTANTS YOU _SHOULD NOT_ HAVE TO CHANGE ----------

# a template for html placeholders
HTML_PLACEHOLDER_PREFIX = "qaodmasdkwaspemas"
HTML_PLACEHOLDER = HTML_PLACEHOLDER_PREFIX + "%dajkqlsmdqpakldnzsdfls"

BLOCK_LEVEL_ELEMENTS = ['p', 'div', 'blockquote', 'pre', 'table',
                        'dl', 'ol', 'ul', 'script', 'noscript',
                        'form', 'fieldset', 'iframe', 'math', 'ins',
                        'del', 'hr', 'hr/', 'style']

def is_block_level (tag) :
    return ( (tag in BLOCK_LEVEL_ELEMENTS) or
             (tag[0] == 'h' and tag[1] in "0123456789") )

"""
======================================================================
========================== NANODOM ===================================
======================================================================

{'fraction_non_alphanumeric': 0.21179426544487143, 'fraction_numerical': 0.030298551581436593, 'mean_word_length': 2.922898550724638, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4170408', 'n_tokens_mistral': 2398, 'n_tokens_neox': 2148, 'n_words': 582}