<commit_before>from app import app
from app.database import db, \
                         Domain, \
                         Whois
from flask.ext.mail import Mail, Message
from pythonwhois.net import get_whois_raw as get_whois
from difflib import unified_diff
from time import sleep
import re

with app.app_context():
    mail = Mail(app)

    while True:
        # Get a list of all domains
        domains = db.session.query(Domain).\
                             order_by(Domain.name).\
                             all()

        # Do a whois lookup on each domain
        for domain in domains:
            print "*** Processing %s" % domain.name

            # Retrieve the stored whois for this domain
            whois = db.session.query(Whois).\
                              filter(Whois.id == domain.whois_id).\
                              first()

            # Lookup the current whois
            whois_fresh = get_whois(domain.name)[0]

            # Filter the whois info for .ca domains
            whois_fresh = re.sub(r'(%.*)\n', '', whois_fresh, flags=re.MULTILINE)

            if whois is None:
                print "Saving new whois information for %s" % domain.name
                # No previous whois found, so just save it
                whois = Whois(domain=domain.name, value=whois_fresh)
                db.session.add(whois)
                db.session.commit()
            elif whois.value is not None and whois.value != whois_fresh:
                print "Whois information has changed for %s" % domain.name

                # Build the email body<commit_msg>Improve filtering for .com domains<commit_after>from app import app
from app.database import db, \
                         Domain, \
                         Whois
from flask.ext.mail import Mail, Message
from pythonwhois.net import get_whois_raw as get_whois
from difflib import unified_diff
from time import sleep
import re

with app.app_context():
    mail = Mail(app)

    while True:
        # Get a list of all domains
        domains = db.session.query(Domain).\
                             order_by(Domain.name).\
                             all()

        # Do a whois lookup on each domain
        for domain in domains:
            print "*** Processing %s" % domain.name

            # Retrieve the stored whois for this domain
            whois = db.session.query(Whois).\
                              filter(Whois.id == domain.whois_id).\
                              first()

            # Lookup the current whois
            whois_fresh = get_whois(domain.name)[0]

            # Filter the whois info for .ca domains
            whois_fresh = re.sub(r'(%.*)\n', '', whois_fresh, flags=re.MULTILINE)

            # Filter the whois info for .com domains
            whois_fresh = re.sub(r'(>>>.*)\n', '', whois_fresh, flags=re.MULTILINE)

            if whois is None:
                print "Saving new whois information for %s" % domain.name
                # No previous whois found, so just save it
                whois = Whois(domain=domain.name, value=whois_fresh)
                db.session.add(whois)
                db.session.commit()
            elif whois.value is not None and whois.value != whois_fresh:
                print "Whois information has changed for %s" % domain.name

                # Build the email body
{'fraction_non_alphanumeric': 0.08181546730367274, 'fraction_numerical': 0.0005971931919976113, 'mean_word_length': 1.5533536585365855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1906827', 'n_tokens_mistral': 938, 'n_tokens_neox': 870, 'n_words': 286}