<commit_before>#!/usr/bin/python
import sys
import os
import StringIO
import time
import codecs
import smtplib
import traceback
from argparse import ArgumentParser
import imaplib
import email
import re
import cgi
import cfgreader

# Read in custom configurations
g_cfg = cfgreader.CfgReader(__file__.replace('.py', '.cfg'))

# These two strings will form the header and individual
# items of the RSS feed.
feed_header = """<?xml version="1.0" encoding="iso-8859-1"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>DVDs shipped for %s</title>
<link>http://dvd.netflix.com/Queue</link>
<atom:link href="http://%s/%s.xml" rel="self" type="application/rss+xml" />
<pubDate>%%s</pubDate>
<description>Feed automatically generated by %s</description>
<language>en-us</language>
""" % (g_cfg.main.name, g_cfg.main.url_base,
       g_cfg.main.rss_base, g_cfg.main.url_base)

feed_item = """<item>
<title>%s</title>
<pubDate>%s</pubDate>
<link>%s</link>
<guid isPermaLink="false">%s</guid>
<description>The disc &lt;a href="%s"&gt;%s&lt;/a&gt; was shipped on %s.</description>
</item>
"""


def set_v_print(verbose):
    """
    Defines the function v_print.
    It prints if verbose is true, otherwise, it does nothing.
    See: http://stackoverflow.com/questions/5980042
    :param verbose: A bool to determine if v_print will print its args.
    """
    global v_print
    if verbose:
        def v_print(*s):
            print ' '.join([i.encode('utf8') for i in s])
    else:
        v_print = lambda *s: None


def send_email(subject, msg, toaddrs,
               fromaddr='"%s" <%s>' % (os.path.basename(__file__),
                                       g_cfg.smtp.from_addr)):
    """ Sends Email
    This function is only used in an emergency.
    """
    smtp = smtplib.SMTP('localhost', port=g_cfg.smtp.port)
    smtp.login(g_cfg.smtp.user, g_cfg.smtp.password)
    smtp.sendmail(fromaddr, toaddrs,
                  "Content-Type: text/plain; charset=\"us-ascii\"\r\n"
                  "From: %s\r\nTo: %s\r\nSubject: %s\r\n%s" %
                  (fromaddr, ", ".join(toaddrs), subject, msg))
    smtp.quit()


def write_feed(script_dir, feed_items):
    """ Given a list of feed_items, write an FSS feed. """
    now = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
    index = 0
    do_move = False
    temp_fname = os.path.join(script_dir, g_cfg.main.rss_base + '.temp.xml')
    dest_fname = os.path.join(script_dir, g_cfg.main.rss_base + '.xml')
    with open(temp_fname, 'wb') as f:
        f.write(feed_header % (now,))
        for title, url, ship_date in reversed(feed_items):
            title = cgi.escape(title)
            guid = "%s+%s+%d" % (url, now, index)
            f.write(feed_item % (title,
                                 ship_date,
                                 url,
                                 guid,
                                 url, title, ship_date[:-15]))
            index += 1
        f.write('</channel></rss>')
        do_move = True
    if do_move:
        os.rename(temp_fname, dest_fname)
        return "OK (Wrote %d new item%s.)" % \
               (len(feed_items), len(feed_items) > 1 and "s" or "")
    return "Could not update the feed file."


def titles_from_text_part(part):
    """ Given a text part of the message, try to get all the titles.
    They'll be the lines after "WE SHIPPED:" and before "WE RECEIVED" or
    "***" """
    titles = list()
    titles_begin = False
    txt = str(part)
    for line in txt.splitlines():
        if line.startswith("WE SHIPPED"):
            titles_begin = True
        elif line.startswith("* Est. arrival") or len(line) < 2:
            continue
        elif line.startswith("WE RECEIVED") or line.startswith("*****"):
            break
        elif titles_begin == True:
            titles.append(line.strip())
            v_print("Found title", line.strip())
    return titles


def get_urls_from_message(part, titles):
    """ Given a part of an email message, try to find the NetFlix URL within. """
    urls = list()
    urlpat = re.compile('http://dvd.netflix.com/Movie/\d+')
    txt = str(part)
    subpart_begin = 0
    for title in titles:
        pos = txt.find(title, subpart_begin)
        if pos == -1:
            return "NO", ["The HTML body no longer has the title in it.",]
        else:
            subpart = txt[subpart_begin:pos]
            subpart_begin = pos
            matches = urlpat.search(subpart)
            if matches is None:
                return "NO", ["The HTML body no longer has the same type of URL.",]
            v_print("Found URL", matches.group(0))
            urls.append(matches.group(0))
    return "OK", urls


def main(script_dir, debug):
    """ Fetch all the mail, and try to find messages that
    match a pattern like, "For Wed: Some Movie".

    It'll add those movies to the RSS feed, and if that's
    completed successfully, then it'll delete the processed email.
    """
    server = imaplib.IMAP4(g_cfg.imap.mailbox)
    server.login(g_cfg.imap.user, g_cfg.imap.password)
    server.select()
    status, data = server.search(None, 'ALL')
    if status != 'OK':
        raise Exception('Getting the list of messages resulted in %s' % status)

    messages_to_delete = []
    feed_items = []
    for num in data[0].split():  # For each email message...
        status, data = server.fetch(num, '(RFC822)')
        if status != 'OK':
            raise Exception('Fetching message %s resulted in %s' % (num, status))
        msg = email.message_from_string(data[0][1])
        subject = msg['Subject']

        if subject.startswith('Fwd: '):
            subject = subject[5:]

        if subject.startswith("We received") or \
           subject.startswith("Shipping today!"):
            # We will bypass, but delete this message as recognized
            v_print("Bypassing message", num, subject)
            messages_to_delete.append(num)
            continue

        if not (subject.startswith("For ") and subject.find(':') != -1) and \
           not subject.startswith("We sent you "):
            print 'Subject "%s" was unexpected, but the script is continuing. ' \
                  'Please take a look at the mailbox.' % subject
            continue<commit_msg>Add more email recognition logic.<commit_after>#!/usr/bin/python
import sys
import os
import StringIO
import time
import codecs
import smtplib
import traceback
from argparse import ArgumentParser
import imaplib
import email
import re
import cgi
import cfgreader

# Read in custom configurations
g_cfg = cfgreader.CfgReader(__file__.replace('.py', '.cfg'))

# These two strings will form the header and individual
# items of the RSS feed.
feed_header = """<?xml version="1.0" encoding="iso-8859-1"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>DVDs shipped for %s</title>
<link>http://dvd.netflix.com/Queue</link>
<atom:link href="http://%s/%s.xml" rel="self" type="application/rss+xml" />
<pubDate>%%s</pubDate>
<description>Feed automatically generated by %s</description>
<language>en-us</language>
""" % (g_cfg.main.name, g_cfg.main.url_base,
       g_cfg.main.rss_base, g_cfg.main.url_base)

feed_item = """<item>
<title>%s</title>
<pubDate>%s</pubDate>
<link>%s</link>
<guid isPermaLink="false">%s</guid>
<description>The disc &lt;a href="%s"&gt;%s&lt;/a&gt; was shipped on %s.</description>
</item>
"""


def set_v_print(verbose):
    """
    Defines the function v_print.
    It prints if verbose is true, otherwise, it does nothing.
    See: http://stackoverflow.com/questions/5980042
    :param verbose: A bool to determine if v_print will print its args.
    """
    global v_print
    if verbose:
        def v_print(*s):
            print ' '.join([i.encode('utf8') for i in s])
    else:
        v_print = lambda *s: None


def send_email(subject, msg, toaddrs,
               fromaddr='"%s" <%s>' % (os.path.basename(__file__),
                                       g_cfg.smtp.from_addr)):
    """ Sends Email
    This function is only used in an emergency.
    """
    smtp = smtplib.SMTP('localhost', port=g_cfg.smtp.port)
    smtp.login(g_cfg.smtp.user, g_cfg.smtp.password)
    smtp.sendmail(fromaddr, toaddrs,
                  "Content-Type: text/plain; charset=\"us-ascii\"\r\n"
                  "From: %s\r\nTo: %s\r\nSubject: %s\r\n%s" %
                  (fromaddr, ", ".join(toaddrs), subject, msg))
    smtp.quit()


def write_feed(script_dir, feed_items):
    """ Given a list of feed_items, write an FSS feed. """
    now = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
    index = 0
    do_move = False
    temp_fname = os.path.join(script_dir, g_cfg.main.rss_base + '.temp.xml')
    dest_fname = os.path.join(script_dir, g_cfg.main.rss_base + '.xml')
    with open(temp_fname, 'wb') as f:
        f.write(feed_header % (now,))
        for title, url, ship_date in reversed(feed_items):
            title = cgi.escape(title)
            guid = "%s+%s+%d" % (url, now, index)
            f.write(feed_item % (title,
                                 ship_date,
                                 url,
                                 guid,
                                 url, title, ship_date[:-15]))
            index += 1
        f.write('</channel></rss>')
        do_move = True
    if do_move:
        os.rename(temp_fname, dest_fname)
        return "OK (Wrote %d new item%s.)" % \
               (len(feed_items), len(feed_items) > 1 and "s" or "")
    return "Could not update the feed file."


def is_line_after_dvd_list(line):
    """ Returns whether this line is known to come after the
    list of titles being shipped. """
    lines = ["WE RECEIVED", "*****", "KEEP SHIPMENTS COMING"]
    return any(line.startswith(words) for words in lines)


def titles_from_text_part(part):
    """ Given a text part of the message, try to get all the titles.
    They'll be the lines after "WE SHIPPED:" and before "WE RECEIVED" or
    "***" """
    titles = list()
    titles_begin = False
    txt = str(part)
    for line in txt.splitlines():
        if line.startswith("WE SHIPPED"):
            titles_begin = True
        elif line.startswith("* Est. arrival") or len(line) < 2:
            continue
        elif is_line_after_dvd_list(line):
            break
        elif titles_begin == True:
            titles.append(line.strip())
            v_print("Found title", line.strip())
    return titles


def get_urls_from_message(part, titles):
    """ Given a part of an email message, try to find the NetFlix URL within. """
    urls = list()
    urlpat = re.compile('http://dvd.netflix.com/Movie/\d+')
    txt = str(part)
    subpart_begin = 0
    for title in titles:
        pos = txt.find(title, subpart_begin)
        if pos == -1:
            return "NO", ['The HTML body did not have the title "%s" in it.' % (title,),]
        else:
            subpart = txt[subpart_begin:pos]
            subpart_begin = pos
            matches = urlpat.search(subpart)
            if matches is None:
                return "NO", ["The HTML body no longer has the same type of URL.",]
            v_print("Found URL", matches.group(0))
            urls.append(matches.group(0))
    return "OK", urls


def subject_is_recognized(subject):
    """ Returns whether this subject line is known to be associated
    with email that contains a title of a DVD to be shipped.
    """
    recognized_subjects = ["We sent you ", "We shipped you "]
    if any(subject.startswith(words) for words in recognized_subjects):
        return True
    return subject.startswith("For ") and subject.find(':') != -1


def main(script_dir, debug):
    """ Fetch all the mail, and try to find messages that
    match a pattern like, "For Wed: Some Movie".

    It'll add those movies to the RSS feed, and if that's
    completed successfully, then it'll delete the processed email.
    """
    server = imaplib.IMAP4(g_cfg.imap.mailbox)
    server.login(g_cfg.imap.user, g_cfg.imap.password)
    server.select()
    status, data = server.search(None, 'ALL')
    if status != 'OK':
        raise Exception('Getting the list of messages resulted in %s' % status)

    messages_to_delete = []
    feed_items = []
    for num in data[0].split():  # For each email message...
        status, data = server.fetch(num, '(RFC822)')
        if status != 'OK':
            raise Exception('Fetching message %s resulted in %s' % (num, status))
        msg = email.message_from_string(data[0][1])
        subject = msg['Subject']

        if subject.startswith('Fwd: '):
            subject = subject[5:]

        if subject.startswith("We received") or \
           subject.startswith("Shipping today!"):
            # We will bypass, but delete this message as recognized
            v_print("Bypassing message", num, subject)
            messages_to_delete.append(num)
            continue

        if not subject_is_recognized(subject):
            print 'Subject "%s" was unexpected, but the script is continuing. ' \
                  'Please take a look at the mailbox.' % subject
            continue
{'fraction_non_alphanumeric': 0.13279524861037081, 'fraction_numerical': 0.006852965811315008, 'mean_word_length': 2.439120188531029, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 2, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1734493', 'n_tokens_mistral': 4267, 'n_tokens_neox': 4158, 'n_words': 1179}