<commit_before>import requests 
import lxml.html
import re
import traceback
import urlparse

_malware_keywords = {
    'Queen Elizabeth II':0
}
    
def check_content(url):
    global _malware_keywords
    try:
        response = requests.get(url, timeout=10.0)
        content = response.content.lower()
        for k,v in _malware_keywords.items():
            _malware_keywords[k] = _malware_keywords[k] + content.count(k)
        return _malware_keywords
    except:
        #raise
        return 'Could not check content of: ' + url



def find_inner_urls(url):
    try:
        response = requests.get(url, timeout=10.0)
        html = lxml.html.fromstring(
            response.content.lower(),
            base_url=response.url
        )
        parsed_url = urlparse.urlparse(response.url)
        base_url = urlparse.urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', ''))
        urls = html.xpath('//a/@href')
        for i, u in enumerate(urls):
            if not u.startswith('http://'):
                urls[i] = base_url + u
        return urls
    except:
        #raise
        return 'Could not find urls for: ' + url

def malware_scan(url):
    global _malware_keywords
    try:
        urls = find_inner_urls(url)
        for i, u in enumerate(urls):
            print ('checking url of: ' + u, str(i)+'/'+str(len(urls)))
            check_content(u)
        return _malware_keywords
        #e_title = lxml.html.fromstring(
            #response.content,
            #base_url=response.url
        #).cssselect("title")
        #if e_title:
            #return e_title[0].text
    except:<commit_msg>Fixed a couple of bugs in checking the content<commit_after>import requests 
import lxml.html
import re
import traceback
import urlparse

_malware_keywords = {
    'Queen Elizabeth II':0,
    'Columbia University':0
}

_lower_mk = {}
for k,v in _malware_keywords.items():
    _lower_mk[k.lower()] = v;

_malware_keywords = _lower_mk
    
def check_content(url):
    global _malware_keywords
    try:
        response = requests.get(url, timeout=10.0)
        content = response.content.lower()
        for k,v in _malware_keywords.items():
            _malware_keywords[k] = _malware_keywords[k] + content.count(k)
        return _malware_keywords
    except:
        #raise
        return 'Could not check content of: ' + url



def find_inner_urls(url):
    try:
        response = requests.get(url, timeout=10.0)
        html = lxml.html.fromstring(
            response.content.lower(),
            base_url=response.url
        )
        parsed_url = urlparse.urlparse(response.url)
        base_url = urlparse.urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', ''))
        urls = html.xpath('//a/@href')
        for i, u in enumerate(urls):
            if not u.startswith('http://'):
                urls[i] = base_url + u
        return urls
    except:
        #raise
        return 'Could not find urls for: ' + url


def malware_scan(url):
    global _malware_keywords
    try:
        urls = find_inner_urls(url)
        for i, u in enumerate(urls):
            print ('checking url of: ' + u, str(i)+'/'+str(len(urls)))
            check_content(u)
        return _malware_keywords
        #e_title = lxml.html.fromstring(
            #response.content,
            #base_url=response.url
        #).cssselect("title")
        #if e_title:
            #return e_title[0].text
    except:
{'fraction_non_alphanumeric': 0.11259431224608242, 'fraction_numerical': 0.004933255948926291, 'mean_word_length': 2.212488350419385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5512901', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1086, 'n_words': 242}