<commit_before>import requests
from urlparse import urlparse

from six.moves import zip, map
from six import binary_type

from celery import task
from celery.contrib.batches import Batches
from celery import current_app

wot_api_target = "https://api.mywot.com/0.4/public_link_json"


@task(base=Batches, flush_every=100, flush_interval=10)
def wot_malware_scan(requests):
    sig = lambda url: url
    reponses = wot_malware_scan_real(
        (sig(*request.args, **request.kwargs) for request in requests)
    )
    # use mark_as_done to manually return response data
    for response, request in zip(reponses, requests):
        current_app.backend.mark_as_done(request.id, response)


def prepare_requests(domains):
    "Insure requests are chopped up to the correct size for the WOT api"
    prepared = list(map(lambda domain: binary_type(domain + '/'), set(domains)))
    while prepared:
        req = binary_type(wot_api_target + "?hosts=")
        while prepared and (len(req) + len(prepared[-1])) <= (8 * 1024):
            req += prepared.pop()
        yield req


def wot_malware_scan_real(urls):
    domains = [urlparse(url).netloc for url in urls]

    total_response = {}
    for request in prepare_requests(domains):
        total_response.update(requests.get(request).json)

    return [total_response[domain] for domain in domains]


if __name__ == "__main__":
    print wot_malware_scan_real(("http://google.com", "http://yahoo.com"))<commit_msg>fix spelling error in prepare_requests docstring<commit_after>import requests
from urlparse import urlparse

from six.moves import zip, map
from six import binary_type

from celery import task
from celery.contrib.batches import Batches
from celery import current_app

wot_api_target = "https://api.mywot.com/0.4/public_link_json"


@task(base=Batches, flush_every=100, flush_interval=10)
def wot_malware_scan(requests):
    sig = lambda url: url
    reponses = wot_malware_scan_real(
        (sig(*request.args, **request.kwargs) for request in requests)
    )
    # use mark_as_done to manually return response data
    for response, request in zip(reponses, requests):
        current_app.backend.mark_as_done(request.id, response)


def prepare_requests(domains):
    "ensure requests to the WOT api are less than 8KiB"
    prepared = list(map(lambda domain: binary_type(domain + '/'), set(domains)))
    while prepared:
        req = binary_type(wot_api_target + "?hosts=")
        while prepared and (len(req) + len(prepared[-1])) <= (8 * 1024):
            req += prepared.pop()
        yield req


def wot_malware_scan_real(urls):
    domains = [urlparse(url).netloc for url in urls]

    total_response = {}
    for request in prepare_requests(domains):
        total_response.update(requests.get(request).json)

    return [total_response[domain] for domain in domains]


if __name__ == "__main__":
    print wot_malware_scan_real(("http://google.com", "http://yahoo.com"))
{'fraction_non_alphanumeric': 0.10454854039375425, 'fraction_numerical': 0.009164969450101833, 'mean_word_length': 4.00339558573854, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6248601', 'n_tokens_mistral': 954, 'n_tokens_neox': 908, 'n_words': 247}