<commit_before># -*- coding: utf-8 -*-

"""
update.py
Jan 14, 2014
Copyright (C) 2014 <NAME>
"""

import os
import redis
import dns.resolver
import urllib2
from main import dns_list

dns_resolver = dns.resolver.Resolver()

redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
redis = redis.from_url(redis_url)

for name_server in dns_list:
    dns_resolver.nameservers = dns_list[name_server]

    try:
        answers = dns_resolver.query('youtube.com')
    except:
        answers = None

    if not answers:
        try:
            redis.set(name_server, 'dns_down')
        except:
            pass
        print name_server, 'is down'
        continue

    address = 'http://' + str(answers[0])

    try:
        response = urllib2.urlopen(address)
    except:
        try:
            redis.set(name_server, 'down')
        except:
            pass
        print address, 'is down when accessed from', name_server
        continue

    for line in response:
        if 'mahkeme' in line.lower():
            try:
                redis.set(name_server, 'court_blocked')
            except:
                pass
            print address, 'seems to be blocked by court order when requested from', name_server
            break

    try:
        redis.set(name_server, 'up')
    except:
        pass
    print address, 'reachable from', name_server<commit_msg>Court block checking bug fixed.<commit_after># -*- coding: utf-8 -*-

"""
update.py
Jan 14, 2014
Copyright (C) 2014 <NAME>
"""

import os
import redis
import dns.resolver
import urllib2
from main import dns_list

dns_resolver = dns.resolver.Resolver()

redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
redis = redis.from_url(redis_url)

for name_server in dns_list:
    dns_resolver.nameservers = dns_list[name_server]

    try:
        answers = dns_resolver.query('youtube.com')
    except:
        answers = None

    if not answers:
        try:
            redis.set(name_server, 'dns_down')
        except:
            pass
        print name_server, 'is down'
        continue

    address = 'http://' + str(answers[0])

    try:
        response = urllib2.urlopen(address)
    except:
        try:
            redis.set(name_server, 'down')
        except:
            pass
        print address, 'is down when accessed from', name_server
        continue

    court_blocked = False

    for line in response:
        if 'mahkeme' in line.lower():
            court_blocked = True
            try:
                redis.set(name_server, 'court_blocked')
            except:
                pass
            print address, 'seems to be blocked by court order when requested from', name_server
            break

    if not court_blocked:
        try:
            redis.set(name_server, 'up')
        except:
            pass
        print address, 'reachable from', name_server
{'fraction_non_alphanumeric': 0.09457579972183588, 'fraction_numerical': 0.012517385257301807, 'mean_word_length': 2.047669491525424, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7403899', 'n_tokens_mistral': 934, 'n_tokens_neox': 891, 'n_words': 236}