archiveis/api.py
def capture(
    target_url,
    user_agent="archiveis (https://github.com/pastpages/archiveis)",
    proxies={}
):
    """
    Archives the provided URL using archive.is

    Returns the URL where the capture is stored.
    """
    # Put together the URL that will save our request
    domain = "http://archive.vn"
    save_url = urljoin(domain, "/submit/")

    # Configure the request headers
    headers = {
        'User-Agent': user_agent,
        "host": "archive.vn",
    }

    # Request a unique identifier for our activity
    logger.debug("Requesting {}".format(domain + "/"))
    get_kwargs = dict(
        timeout=120,
        allow_redirects=True,
        headers=headers,
    )
    if proxies:
        get_kwargs['proxies'] = proxies
    response = requests.get(domain + "/", **get_kwargs)
    response.raise_for_status()

    # It will need to be parsed from the homepage response headers
    html = str(response.content)
    try:
        unique_id = html.split('name="submitid', 1)[1].split('value="', 1)[1].split('"', 1)[0]
        logger.debug("Unique identifier: {}".format(unique_id))
    except IndexError:
        logger.warn("Unable to extract unique identifier from archive.is. Submitting without it.")
        unique_id = None

    # Send the capture request to archive.is with the unique id included
    data = {
        "url": target_url,
        "anyway": 1,
    }
    if unique_id:
        data.update({"submitid": unique_id})

    post_kwargs = dict(
        timeout=120,
        allow_redirects=True,
        headers=headers,
        data=data
    )
    if proxies:
        post_kwargs['proxies'] = proxies

    logger.debug("Requesting {}".format(save_url))
    response = requests.post(save_url, **post_kwargs)
    response.raise_for_status()

    # There are a couple ways the header can come back
    if 'Refresh' in response.headers:
        memento = str(response.headers['Refresh']).split(';url=')[1]
        logger.debug("Memento from Refresh header: {}".format(memento))
        return memento
    if 'Location' in response.headers:
        memento = response.headers['Location']
        logger.debug("Memento from Location header: {}".format(memento))
        return memento
    logger.debug("Memento not found in response headers. Inspecting history.")
    for i, r in enumerate(response.history):
        logger.debug("Inspecting history request #{}".format(i))
        logger.debug(r.headers)
        if 'Location' in r.headers:
            memento = r.headers['Location']
            logger.debug("Memento from the Location header of {} history response: {}".format(i+1, memento))
            return memento
    # If there's nothing at this point, throw an error
    logger.error("No memento returned by archive.is")
    logger.error("Status code: {}".format(response.status_code))
    logger.error(response.headers)
    logger.error(response.text)
    raise Exception("No memento returned by archive.is")
def capture(
    target_url,
    user_agent="archiveis (https://github.com/pastpages/archiveis)",
    proxies={}
):
    """
    Archives the provided URL using archive.is

    Returns the URL where the capture is stored.
    """
    # Put together the URL that will save our request
    domain = "http://archive.vn"
    save_url = urljoin(domain, "/submit/")

    # Configure the request headers
    headers = {
        'User-Agent': user_agent,
        "host": "archive.vn",
    }

    # Request a unique identifier for our activity
    logger.debug("Requesting {}".format(domain + "/"))
    get_kwargs = dict(
        timeout=120,
        allow_redirects=True,
        headers=headers,
    )
    if proxies:
        get_kwargs['proxies'] = proxies
    response = requests.get(domain + "/", **get_kwargs)
    response.raise_for_status()

    # It will need to be parsed from the homepage response headers
    html = str(response.content)
    try:
        unique_id = html.split('name="submitid', 1)[1].split('value="', 1)[1].split('"', 1)[0]
        logger.debug("Unique identifier: {}".format(unique_id))
    except IndexError:
        logger.warn("Unable to extract unique identifier from archive.is. Submitting without it.")
        unique_id = None

    # Send the capture request to archive.is with the unique id included
    data = {
        "url": target_url,
        "anyway": 1,
    }
    if unique_id:
        data.update({"submitid": unique_id})

    post_kwargs = dict(
        timeout=120,
        allow_redirects=True,
        headers=headers,
        data=data
    )
    if proxies:
        post_kwargs['proxies'] = proxies

    logger.debug("Requesting {}".format(save_url))
    response = requests.post(save_url, **post_kwargs)
    response.raise_for_status()

    # There are a couple ways the header can come back
    if 'Refresh' in response.headers:
        memento = str(response.headers['Refresh']).split(';url=')[1]
        logger.debug("Memento from Refresh header: {}".format(memento))
        return memento
    if 'Location' in response.headers:
        memento = response.headers['Location']
        logger.debug("Memento from Location header: {}".format(memento))
        return memento
    logger.debug("Memento not found in response headers. Inspecting history.")
    for i, r in enumerate(response.history):
        logger.debug("Inspecting history request #{}".format(i))
        logger.debug(r.headers)
        if 'Location' in r.headers:
            memento = r.headers['Location']
            logger.debug("Memento from the Location header of {} history response: {}".format(i+1, memento))
            return memento
    # If there's nothing at this point, throw an error
    logger.error("No memento returned by archive.is")
    logger.error("Status code: {}".format(response.status_code))
    logger.error(response.headers)
    logger.error(response.text)
    raise Exception("No memento returned by archive.is")
Archives the provided URL using archive.is

    Returns the URL where the capture is stored.