<commit_before>#!/usr/bin/python
"""
Test all the links in the CSV for non-200 status codes (after redirects).
"""

import sys
import urllib3

from django.core.management.base import BaseCommand, CommandError
from myuw.dao.category_links import Res_Links

# Disable SSL warnings
urllib3.disable_warnings()
# Need limit of 1, otherwise sdb gives us a 403
http = urllib3.PoolManager(1, timeout=8)
# Need to override UA for some links, e.g. LinkedIn
user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'


class Command(BaseCommand):

    help = ('Test all resource links for non-200 status codes after'
    ' following redirects')

    def handle(self, *args, **kwargs):
        try:
            check_all()
        except Exception as e:
            raise CommandError(e)


def get_http_status(url):
    """
    Given a url, get the HTTP status code or a human-readable exception.
    """
    try:
        result = http.request(
            'GET',
            url,
            headers={'User-Agent': user_agent},
            retries = urllib3.Retry(redirect=10, connect=2, read=2)
        )
        return result.status

    except Exception:
        return repr(sys.exc_info()[1])


def check_and_format(link, ignore=[200]):
    result = get_http_status(link.url)

    if result in ignore:
        return None

    else:
        campus = campus_human_readable(link.campus)
        fmt = (campus, link.title, link.url, result)
        return 'URL (%s) %s (%s) returned %s' % fmt


def check_all():

    links = Res_Links.get_all_links()
    for link in links:
        out = check_and_format(link)
        if out is not None:
            print out


def campus_human_readable(campus):

    if campus is None:
        return 'All'
    else:
        # Capitalize first letter
        return campus[0:1].upper() + campus[1:]


if __name__ == '__main__':
    check_all()<commit_msg>Removed old code from link checker<commit_after>#!/usr/bin/python
"""
Test all the links in the CSV for non-200 status codes (after redirects).
"""

import sys
import urllib3

from django.core.management.base import BaseCommand, CommandError
from myuw.dao.category_links import Res_Links

# Disable SSL warnings
urllib3.disable_warnings()
# Need limit of 1, otherwise sdb gives us a 403
http = urllib3.PoolManager(1, timeout=8)
# Need to override UA for some links, e.g. LinkedIn
user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'


class Command(BaseCommand):

    help = ('Test all resource links for non-200 status codes after'
    ' following redirects')

    def handle(self, *args, **kwargs):
        try:
            check_all()
        except Exception as e:
            raise CommandError(e)


def get_http_status(url):
    """
    Given a url, get the HTTP status code or a human-readable exception.
    """
    try:
        result = http.request(
            'GET',
            url,
            headers={'User-Agent': user_agent},
            retries = urllib3.Retry(redirect=10, connect=2, read=2)
        )
        return result.status

    except Exception:
        return repr(sys.exc_info()[1])


def check_and_format(link, ignore=[200]):
    result = get_http_status(link.url)

    if result in ignore:
        return None

    else:
        campus = campus_human_readable(link.campus)
        fmt = (campus, link.title, link.url, result)
        return 'URL (%s) %s (%s) returned %s' % fmt


def check_all():

    links = Res_Links.get_all_links()
    for link in links:
        out = check_and_format(link)
        if out is not None:
            print out


def campus_human_readable(campus):

    if campus is None:
        return 'All'
    else:
        # Capitalize first letter
        return campus[0:1].upper() + campus[1:]
{'fraction_non_alphanumeric': 0.10322069693769799, 'fraction_numerical': 0.02587117212249208, 'mean_word_length': 2.6120114394661584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6597282', 'n_tokens_mistral': 1261, 'n_tokens_neox': 1179, 'n_words': 362}