<commit_before>from __future__ import print_function, unicode_literals
import requests
import lxml.html
import six
from progress.bar import Bar


<<<<<<< HEAD
_malware_keywords = {'Queen Elizabeth II',}
_lower_mk = {}
for k,v in _malware_keywords.items():
    _lower_mk[k.lower()] = v;
=======
_malware_keywords = { keyword.lower() for keyword in ('Queen Elizabeth II','Columbia University') }
>>>>>>> feccc89c90bce610e05d01e0f2522834cd1fa40e

_malware_keywords = _lower_mk

def check_content(url):
    try:
        response = requests.get(url, timeout=10.0)
        response.raise_for_status()
    except requests.exceptions.HTTPError:
        return {}

    text = response.text.lower()
<<<<<<< HEAD
    return {keyword: text.count(keyword) for keyword in _malware_keywords} 
=======
    return {keyword: text.count(keyword) for keyword in _malware_keywords}
>>>>>>> feccc89c90bce610e05d01e0f2522834cd1fa40e


def malware_scan(url):
    """
    Crawls a page depth 1, returns the total count for each of the keywords in _malware_keywords for each of the pages in the crawl.
    """

    response = requests.get(url, timeout=10.0)
    html = lxml.html.fromstring(
        response.content,
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)

    keyword_counts = dict.fromkeys(_malware_keywords, 0)

    for url in Bar().iter({link for element, attribute, link, pos in html.iterlinks()}):
        for keyword, count in six.iteritems(check_content(url)):
            keyword_counts[keyword] += count

    if sum(six.itervalues(keyword_counts)) > 20:
        return 1
    else:
        return 0


def manual_html_malware_scan(urls):
    for item in urls:
        item[opinion].append({"type": "placeholder", "confidence": 1.0})
    return urls

if __name__ == '__main__':
    url = 'http://uk.movies.yahoo.com/person/emma-watson/'
    print(malware_scan(url))
<commit_msg>Fixed the merge bit<commit_after>from __future__ import print_function, unicode_literals
import requests
import lxml.html
import six
from progress.bar import Bar

_malware_keywords = {'Queen Elizabeth II':0,}
_lower_mk = {}
for k,v in _malware_keywords.items():
    _lower_mk[k.lower()] = v;
_malware_keywords = _lower_mk

def check_content(url):
    try:
        response = requests.get(url, timeout=10.0)
        response.raise_for_status()
    except requests.exceptions.HTTPError:
        return {}

    text = response.text.lower()
    return {keyword: text.count(keyword) for keyword in _malware_keywords} 


def malware_scan(url):
    """
    Crawls a page depth 1, returns the total count for each of the keywords in _malware_keywords for each of the pages in the crawl.
    """

    response = requests.get(url, timeout=10.0)
    html = lxml.html.fromstring(
        response.content,
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)

    keyword_counts = dict.fromkeys(_malware_keywords, 0)

    for url in Bar().iter({link for element, attribute, link, pos in html.iterlinks()}):
        for keyword, count in six.iteritems(check_content(url)):
            keyword_counts[keyword] += count

    if sum(six.itervalues(keyword_counts)) > 20:
        return 1
    else:
        return 0


def manual_html_malware_scan(urls):
    for item in urls:
        item[opinion].append({"type": "placeholder", "confidence": 1.0})
    return urls

if __name__ == '__main__':
    url = 'http://uk.movies.yahoo.com/person/emma-watson/'
    print(malware_scan(url))

{'fraction_non_alphanumeric': 0.11456255343402678, 'fraction_numerical': 0.02080364776289541, 'mean_word_length': 3.624505928853755, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5304193', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1126, 'n_words': 279}