<commit_before>#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re
from functools import partial
from itertools import cycle


class LinkFormatter(object):
    _image_exts = ('.jpg', '.jpeg', '.png', '.gif', '.bmp')
    _link = re.compile('\[(.+?\|.*?)\]')
    # Based on public domain URL matching regexp available and explained at
    # http://daringfireball.net/2010/07/improved_regex_for_matching_urls
    # Main differences compared to the original:
    # - Protocol is required and it must contain ://
    _url = re.compile(ur'''
\b
(                           # Capture 1: entire matched URL
  [a-z][\w-]+://                # URL protocol and ://
  (?:                           # One or more:
    [^\s()<>]+                      # Run of non-space, non-()<>
    |                               #   or
    \(([^\s()<>]+|(\([^\s()<>]+\)))*\)  # balanced parens, up to 2 levels
  )+
  (?:                           # End with:
    \(([^\s()<>]+|(\([^\s()<>]+\)))*\)  # balanced parens, up to 2 levels
    |                                   #   or
    [^\s`!()\[\]{};:'".,<>?\xab\xbb\u201c\u201d\u2018\u2019]        # not a space or one of these punct chars
  )
)''', re.VERBOSE|re.IGNORECASE)

    def format_url(self, text):
        return self._format_url(text, format_as_image=False)

    def _format_url(self, text, format_as_image=True):
        if '://' not in text:
            return text
        return self._url.sub(partial(self._replace_url, format_as_image), text)

    def _replace_url(self, format_as_image, match):
        url = match.group(1)
        if format_as_image and self._is_image(url):
            return self._get_image(url)
        return self._get_link(url)

    def _get_image(self, src, title=None):<commit_msg>URL matcher: Tuned regexp<commit_after>#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re
from functools import partial
from itertools import cycle


class LinkFormatter(object):
    _image_exts = ('.jpg', '.jpeg', '.png', '.gif', '.bmp')
    _link = re.compile('\[(.+?\|.*?)\]')
    # Based on public domain URL matching regexp available and explained at
    # http://daringfireball.net/2010/07/improved_regex_for_matching_urls
    # Main differences compared to the original:
    # - Protocol is required and it must contain ://
    _url = re.compile(ur'''
\b
(                           # Capture 1: entire matched URL
  [a-z][\w-]+://                # URL protocol and ://
  (?:                           # One or more:
    [^\s()]+                        # Run of non-space, non-()
    |                               #   or
    \((?:[^\s()]+|\([^\s()]+\))*\)  # balanced parens, up to 2 levels
  )+
  (?:                           # End with:
                                    # not a space or one of these punct chars
    [^\s`!()\[\]{};:'".,?\xab\xbb\u201c\u201d\u2018\u2019]
    |                               #   or
    \((?:[^\s()]+|\([^\s()]+\))*\)  # balanced parens, up to 2 levels
  )
)''', re.VERBOSE|re.IGNORECASE)

    def format_url(self, text):
        return self._format_url(text, format_as_image=False)

    def _format_url(self, text, format_as_image=True):
        if '://' not in text:
            return text
        return self._url.sub(partial(self._replace_url, format_as_image), text)

    def _replace_url(self, format_as_image, match):
        url = match.group(1)
        if format_as_image and self._is_image(url):
            return self._get_image(url)
        return self._get_link(url)

    def _get_image(self, src, title=None):
{'fraction_non_alphanumeric': 0.13621691413815365, 'fraction_numerical': 0.015493867010974823, 'mean_word_length': 2.2526242127361793, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5088348', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1408, 'n_words': 436}