# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


from urlparse import urlparse, parse_qs
from celery import task, chord, group, chain
from six.moves import map
from django.conf import settings
from malware_crawl.defaults import REQUESTS_SESSIONS
import lxml.html
import itertools
import random


def dogpile_link_handle(url):
    # parse the click handler from dogpile to get the real URL
    return parse_qs(urlparse(url).query)["du"][0]


@task
def dogpile_multilink(keyword, qsi):
    response = requests.get(
        "http://www.dogpile.co.uk/search/web",
        params={"q": keyword, "qsi": qsi}
    )

    html = lxml.html.fromstring(
        response.content,
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)

    return [
        dogpile_link_handle(anchor.get("href")) for anchor in html.cssselect(
            ".webResult .resultDisplayUrl"
        )
    ]


@task
def tchain(iterable):
    if not NERFED:

# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


# Path: malware_crawl/search.py
from urlparse import urlparse, parse_qs
from celery import task, chord, group, chain
from six.moves import map
from django.conf import settings
from malware_crawl.defaults import REQUESTS_SESSIONS
import lxml.html
import itertools
import random



def dogpile_link_handle(url):
    # parse the click handler from dogpile to get the real URL
    return parse_qs(urlparse(url).query)["du"][0]


@task
def dogpile_multilink(keyword, qsi):
    response = requests.get(
        "http://www.dogpile.co.uk/search/web",
        params={"q": keyword, "qsi": qsi}
    )

    html = lxml.html.fromstring(
        response.content,
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)

    return [
        dogpile_link_handle(anchor.get("href")) for anchor in html.cssselect(
            ".webResult .resultDisplayUrl"
        )
    ]


@task
def tchain(iterable):
    if not NERFED:

        return list(itertools.chain.from_iterable(iterable))