<commit_before>tree = lxml.html.fromstring(re.sub('\s+', ' ', str(html)))

tree2 = lxml.html.fromstring(re.sub('\s+', ' ', html2))

import selenium
from selenium import webdriver

import lxml.html



import lxml.html.diff as df



from urllib.request import urlopen

from selenium.webdriver.common.keys import Keys

tree2 = lxml.html.fromstring(html)



def viewString(x, driver):
    with open('/tmp/seleniumStringPage.html', 'w') as f:
        f.write(x)
    driver.get('file:///tmp/seleniumStringPage.html')

def testUrl(url): 
    try:
        driver = webdriver.Firefox()
        driver.maximize_window()
        driver.get(url)
        html1 = str(urlopen(url).read()).replace('\\n','').replace('\\r','').replace("\\'", "'").replace("\\t", "\t")
        html2 = str(driver.page_source) 
        tree = lxml.html.fromstring(html1)
        tree2 = lxml.html.fromstring(html2)
        diffHtml = df.htmldiff(tree, tree2)
        diffTree = lxml.html.fromstring(diffHtml)
        insCounts = diffTree.xpath('count(//ins)')
        delCounts = diffTree.xpath('count(//del)')
        pureDiff = ''
        for x in ['.//ins', './/del']:
            for y in [z for z in diffTree.iter() if z.tag in ['ins', 'del']]:
                if y.text is not None:
                    color = 'lightgreen' if 'ins' in y.tag else 'red'
                    pureDiff += '<div style="background-color:{};">{}</div>'.format(color, y.text)
        print('From non-javascript to javascript, {} insertions and {} deleted'.format(insCounts, delCounts))
        diff = '<head><title>diff</title><base href=' + url + ' target="_blank"><style>ins{ background-color:lightgreen; } del{background-color:red;}</style></head>' +  diffHtml
        viewString(diff, driver) 
        driver.find_element_by_tag_name("body").send_keys(Keys.COMMAND + 't')
        viewString(html1, driver) 
        driver.find_element_by_tag_name("body").send_keys(Keys.COMMAND + 't')
        viewString(html2, driver) 
        driver.find_element_by_tag_name("body").send_keys(Keys.COMMAND + 't')
        viewString('<html><body>{}</body></html>'.format(str(pureDiff)), driver) 
    finally:    
        input('bla?')
        driver.close()

url = 'http://www.healthgrades.com/physician/dr-jeannine-villella-y4jts'
z=testUrl(url)    


attrs = ' '.join(y.keys())<commit_msg>comparison<commit_after>############################ tree = lxml.html.fromstring(re.sub('\s+', ' ', str(html)))

############################ tree2 = lxml.html.fromstring(re.sub('\s+', ' ', html2))

import selenium
from selenium import webdriver
import lxml.html
import lxml.html.diff
from requests import urlopen
from selenium.webdriver.common.keys import Keys

def viewString(x, driver):
    with open('/tmp/seleniumStringPage.html', 'w') as f:
        f.write(x)
    driver.get('file:///tmp/seleniumStringPage.html')

def compareRequestsAndSelenium(url): 
    html1 = str(requests.get(url).text)
    try: 
        driver = webdriver.PhantomJS()
        driver.maximize_window() 
        driver.get(url) 
        html2 = str(driver.page_source) 
    finally:    
        driver.close()
    viewDiffHtml(url, html1, html2)    

def viewDiffHtml(url, html1, html2, diffMethod = lxml.html.diff.htmldiff):
    tree = lxml.html.fromstring(html1)
    tree2 = lxml.html.fromstring(html2)
    diffHtml = diffMethod(tree, tree2)
    diffTree = lxml.html.fromstring(diffHtml)
    insCounts = diffTree.xpath('count(//ins)')
    delCounts = diffTree.xpath('count(//del)')
    pureDiff = ''
    for x in ['.//ins', './/del']:
        for y in [z for z in diffTree.iter() if z.tag in ['ins', 'del']]:
            if y.text is not None:
                color = 'lightgreen' if 'ins' in y.tag else 'red'
                pureDiff += '<div style="background-color:{};">{}</div>'.format(color, y.text)
    print('From non-javascript to javascript, {} insertions and {} deleted'.format(insCounts, delCounts))
    try:    
        driver = webdriver.Firefox()
        diff = '<head><title>diff</title><base href=' + url + ' target="_blank"><style>ins{ background-color:lightgreen; } del{background-color:red;}</style></head>' +  diffHtml
        viewString(diff, driver) 
        driver.find_element_by_tag_name("body").send_keys(Keys.COMMAND + 't')
        viewString(html1, driver) 
        driver.find_element_by_tag_name("body").send_keys(Keys.COMMAND + 't')
        viewString(html2, driver) 
        driver.find_element_by_tag_name("body").send_keys(Keys.COMMAND + 't')
        viewString('<html><body>{}</body></html>'.format(str(pureDiff)), driver) 
    finally:
        input('bla?') 
        driver.close()

url = 'http://www.healthgrades.com/physician/dr-jeannine-villella-y4jts'
z=compareRequestsAndSelenium(url)    



attrs = ' '.join(y.keys())
{'fraction_non_alphanumeric': 0.17628541448058763, 'fraction_numerical': 0.005666316894018888, 'mean_word_length': 3.1157167530224523, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2834581', 'n_tokens_mistral': 1618, 'n_tokens_neox': 1573, 'n_words': 284}