<commit_before>"""
Autolink Extension
==================

A Python-Markdown extension to automatically create links in text.
Initial goal is to mimic GitHub-flavored Markdown:

  - If you see a link, make it work
  - If it's inside of `backticks`, don't

Copyright 2015 <NAME>

License: [MIT](http://opensource.org/licenses/MIT)
"""

import re
from markdown.preprocessors import Preprocessor
from markdown.extensions import Extension

"""
# Old regex is really long and caused problems
# Retained here for reference
urlfinder = re.compile(r'((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+(:[0-9]+)?|'
                       r'(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:/[\+~%/\.\w\-_]*)?\??'
                       r'(?:[\-\+=&;%@\.\w_]*)#?(?:[\.!/\\\w]*))?)')
"""

urlfinder = re.compile(r'(https?:\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?)')

# TODO: Terminate the URL regex if there is a period and immediately after is whitespace or linebreak.
<commit_msg>Whooo autolinks working now :)<commit_after>"""
Autolink Extension
==================

A Python-Markdown extension to automatically create links in text.
Initial goal is to mimic GitHub-flavored Markdown:

  - If you see a link, make it work
  - If it's inside of `backticks`, don't
"""

import re
from markdown.preprocessors import Preprocessor
from markdown.extensions import Extension

"""
# Old regex is really long and caused problems
# Retained here for reference
urlfinder = re.compile(r'((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+(:[0-9]+)?|'
                       r'(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:/[\+~%/\.\w\-_]*)?\??'
                       r'(?:[\-\+=&;%@\.\w_]*)#?(?:[\.!/\\\w]*))?)')
"""

urlfinder = re.compile(r'(https?:\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.\-#?=&%]*[^\.)\s])\/?)')

# What it's doing. This is descriptive, not necessarily prescriptive behavior.
# Matches only URLs prepended with http:// or https://
# Anything that looks like a domain.name
# Any combination of URL-friendly characters, but a non-exhaustive list:
#  - inner periods are OK
#  - dashes are OK
#  - # for anchor hashes
#  - ?, =, & for query strings
#  - % for escaped characters
#  - But NOT any period or closing parenthesis (from Markdown) followed by whitespace
# Trailing slashes are OK

{'fraction_non_alphanumeric': 0.2592433361994841, 'fraction_numerical': 0.010318142734307825, 'mean_word_length': 3.06108202443281, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2349293', 'n_tokens_mistral': 871, 'n_tokens_neox': 821, 'n_words': 213}