<commit_before>import os
import re
import sys
import time
import random
import hashlib
import smtplib
import argparse
import memcache
import datetime
import itertools
import subprocess

from bs4 import BeautifulSoup
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate

reload(sys)
sys.setdefaultencoding('utf8')


class AmzMechanize(object):
    """ Use Python mechanize to login and retrieve URLs. Broken as of May 2016. """
    headers = [
        # ('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.13) '
        #                'Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13')
        ('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 '
                       '(KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36')
    ]

    def __init__(self):
        import mechanize
        self.br = mechanize.Browser()
        self.br.set_handle_robots(False)
        self.br.addheaders = self.headers

    def login(self, email, password):
        self.br.open('https://www.amazon.com')
        resp = self.br.follow_link(text_regex='Sign in')
        self.br.select_form(nr=0)
        self.br['email'] = email
        self.br['password'] = password
        resp = self.br.submit()
        if resp.code != 200:
            raise Exception('Got invalid response code %s' % resp.code)
        elif resp.geturl().startswith('https://www.amazon.com/ap/signin'):  # not the URL we wanted
            html = resp.get_data()
            soup = BeautifulSoup(html, 'lxml')
            err = soup.find_all('div', attrs={'id': 'message_error'})
            warn = soup.find_all('div', attrs={'class': 'a-alert-content'})
            msg = (err and err[0].renderContents()
                   or (warn and warn[0].renderContents())
                   or html).strip()
            raise Exception('Login failed for %s, %s:\n%s' % (email, password, msg))

    def get_url(self, url):
        return self.br.open(url).get_data()

    def clean_up(self):
        pass


class AmzChromeDriver(object):
    """ Replacement driver to login to Amazon and download URLs using the Selenium ChromeDriver. """
    def __init__(self):
        from selenium import webdriver
        self.driver = webdriver.Chrome('/usr/local/bin/chromedriver')
        self.driver.implicitly_wait(30)

    def login(self, email, password):
        driver = self.driver
        driver.get("https://www.amazon.com/")
        driver.find_element_by_css_selector("#nav-signin-tooltip > a.nav-action-button").click()
        driver.find_element_by_id("ap_email").clear()
        driver.find_element_by_id("ap_email").send_keys(email)
        driver.find_element_by_id("ap_password").clear()
        driver.find_element_by_id("ap_password").send_keys(password)
        driver.find_element_by_id("signInSubmit").click()

    def get_url(self, url):
        self.driver.get(url)
        return self.driver.page_source

    def clean_up(self):
        self.driver.quit()


class Emailer(object):

    def __init__(self, smtp_host, smtp_port, smtp_user, smtp_password):<commit_msg>update AmzChromeDriver with latest UI changes<commit_after>import os
import re
import sys
import time
import random
import hashlib
import smtplib
import argparse
import memcache
import datetime
import itertools
import subprocess

from bs4 import BeautifulSoup
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate

reload(sys)
sys.setdefaultencoding('utf8')


class AmzMechanize(object):
    """ Use Python mechanize to login and retrieve URLs. Broken as of May 2016. """
    headers = [
        # ('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.13) '
        #                'Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13')
        ('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 '
                       '(KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36')
    ]

    def __init__(self):
        import mechanize
        self.br = mechanize.Browser()
        self.br.set_handle_robots(False)
        self.br.addheaders = self.headers

    def login(self, email, password):
        self.br.open('https://www.amazon.com')
        resp = self.br.follow_link(text_regex='Hello. Sign in')
        self.br.select_form(nr=0)
        self.br['email'] = email
        self.br['password'] = password
        resp = self.br.submit()
        if resp.code != 200:
            raise Exception('Got invalid response code %s' % resp.code)
        elif resp.geturl().startswith('https://www.amazon.com/ap/signin'):  # not the URL we wanted
            html = resp.get_data()
            soup = BeautifulSoup(html, 'lxml')
            err = soup.find_all('div', attrs={'id': 'message_error'})
            warn = soup.find_all('div', attrs={'class': 'a-alert-content'})
            msg = (err and err[0].renderContents()
                   or (warn and warn[0].renderContents())
                   or html).strip()
            raise Exception('Login failed for %s, %s:\n%s' % (email, password, msg))

    def get_url(self, url):
        return self.br.open(url).get_data()

    def clean_up(self):
        pass


class AmzChromeDriver(object):
    """ Replacement driver to login to Amazon and download URLs using the Selenium ChromeDriver. """
    def __init__(self):
        from selenium import webdriver
        self.driver = webdriver.Chrome('/usr/local/bin/chromedriver')
        self.driver.implicitly_wait(30)

    def login(self, email, password):
        driver = self.driver
        driver.get("https://www.amazon.com/")
        driver.find_element_by_css_selector("#nav-signin-tooltip > a.nav-action-button").click()
        driver.find_element_by_id("ap_email").clear()
        driver.find_element_by_id("ap_email").send_keys(email)
        driver.find_element_by_id("continue").click()
        driver.find_element_by_id("ap_password").clear()
        driver.find_element_by_id("ap_password").send_keys(password)
        driver.find_element_by_id("signInSubmit").click()

    def get_url(self, url):
        self.driver.get(url)
        return self.driver.page_source

    def clean_up(self):
        self.driver.quit()


class Emailer(object):

    def __init__(self, smtp_host, smtp_port, smtp_user, smtp_password):
{'fraction_non_alphanumeric': 0.12344732403005675, 'fraction_numerical': 0.021162398405152584, 'mean_word_length': 2.8118059614260664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4338961', 'n_tokens_mistral': 2150, 'n_tokens_neox': 2029, 'n_words': 456}