<commit_before># -*- coding: utf-8 -*-
"""MSC404: Checks that links are viable.

---
layout:     post
error_code: MSC404
source:     SublimeLinter-annotations
source_url: http://bit.ly/16Q7H41
title:      broken links
date:       2014-06-10 12:31:19
categories: writing
---

Check that links are not not broken.

"""
from proselint.tools import memoize
import re
import urllib2


@memoize
def check(text):
    """Check the text."""
    err = "ANN100"
    msg = u"Broken link: {}"

    regex = re.compile(
        r"""(?i)\b((?:https?://|www\d{0,3}[.]
        |[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+
        |(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)
        |[^\s`!()\[\]{};:\'".,<>?\xab\xbb\u201c\u201d\u2018\u2019]))""",
        re.U)

    errors = []
    for m in re.finditer(regex, text):
        url = m.group(0).strip()

        if "http://" not in url and "https://" not in url:
            url = "http://" + url

        if is_broken_link(url):
            errors.append((m.start(), m.end(), err, msg.format(url)))

    return errors


@memoize
def is_broken_link(url):
    """Check if the link return a 404 error."""
    try:
        urllib2.urlopen(url)
        return False
    except urllib2.URLError:
        return True<commit_msg>Improves link checker<commit_after># -*- coding: utf-8 -*-
"""MSC404: Checks that links are viable.

---
layout:     post
error_code: MSC404
source:     SublimeLinter-annotations
source_url: http://bit.ly/16Q7H41
title:      broken links
date:       2014-06-10 12:31:19
categories: writing
---

Check that links are not not broken.

"""
from proselint.tools import memoize
import re
import urllib2
from socket import error as SocketError


@memoize
def check(text):
    """Check the text."""
    err = "ANN100"
    msg = u"Broken link: {}"

    regex = re.compile(
        ur"""(?i)\b((?:https?://|www\d{0,3}[.]
        |[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+
        |(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)
        |[^\s`!()\[\]{};:\'".,<>?\xab\xbb\u201c\u201d\u2018\u2019\u21a9]))""",
        re.U)

    errors = []
    for m in re.finditer(regex, text):
        url = m.group(0).strip()

        if "http://" not in url and "https://" not in url:
            url = "http://" + url

        if is_broken_link(url):
            errors.append((m.start(), m.end(), err, msg.format(url)))

    return errors


@memoize
def is_broken_link(url):
    """Check if the link return a 404 error."""
    try:
        urllib2.urlopen(url, timeout=2)
        return False
    except urllib2.URLError:
        return True
    except SocketError:
        return True
{'fraction_non_alphanumeric': 0.22788605697151423, 'fraction_numerical': 0.043478260869565216, 'mean_word_length': 2.4, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1769583', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1002, 'n_words': 224}