<commit_before>from __future__ import print_function, unicode_literals
import requests
import lxml.html
import six

_malware_keywords = {
    'Queen Elizabeth II': 0
}


def check_content(url):
    global _malware_keywords
    try:
        response = requests.get(url, timeout=10.0)
        content = response.content.lower()
        for key in six.iterkeys(_malware_keywords):
            _malware_keywords[key] += content.count(key)
        return _malware_keywords
    except:
        #raise
        return 'Could not check content of: ' + url


def malware_scan(url):
    """
    Crawls a page, and adds to total number of occurances of keywords in each page crawled to _malware_keywords
    """

    response = requests.get(url, timeout=10.0)
    html = lxml.html.fromstring(
        response.content.lower(),
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)
    urls = tuple(link for element, attribute, link, pos in html.iterlinks())

    for index, url in enumerate(urls):
        print("checking url: {url}. {index} of {total}".format(url=url, index=index, total=len(urls)))
        check_content(url)

    return _malware_keywords


if __name__ == '__main__':
    url = 'http://uk.movies.yahoo.com/person/emma-watson/'
    print(malware_scan(url))<commit_msg>use set comprehension to cut out duplicates<commit_after>from __future__ import print_function, unicode_literals
import requests
import lxml.html
import six

_malware_keywords = {
    'Queen Elizabeth II': 0
}


def check_content(url):
    global _malware_keywords
    try:
        response = requests.get(url, timeout=10.0)
        content = response.content.lower()
        for key in six.iterkeys(_malware_keywords):
            _malware_keywords[key] += content.count(key)
        return _malware_keywords
    except:
        #raise
        return 'Could not check content of: ' + url


def malware_scan(url):
    """
    Crawls a page, and adds to total number of occurances of keywords in each page crawled to _malware_keywords
    """

    response = requests.get(url, timeout=10.0)
    html = lxml.html.fromstring(
        response.content.lower(),
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)
    urls = {link for element, attribute, link, pos in html.iterlinks()}

    for index, url in enumerate(urls):
        print("checking url: {url}. {index} of {total}".format(url=url, index=index, total=len(urls)))
        check_content(url)

    return _malware_keywords


if __name__ == '__main__':
    url = 'http://uk.movies.yahoo.com/person/emma-watson/'
    print(malware_scan(url))
{'fraction_non_alphanumeric': 0.10291335603480893, 'fraction_numerical': 0.0052970109723798715, 'mean_word_length': 3.2169059011164274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2626936', 'n_tokens_mistral': 853, 'n_tokens_neox': 806, 'n_words': 209}