<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import click
import logging
import requests
from six.moves.urllib.parse import urljoin
logger = logging.getLogger(__name__)


def capture(
    target_url,
    user_agent="archiveis (https://github.com/pastpages/archiveis)",
):
    """
    Archives the provided URL using archive.is

    Returns the URL where the capture is stored.
    """
    # Put together the URL that will save our request
    domain = "http://172.16.17.32"
    save_url = urljoin(domain, "/submit/")

    # Configure the request headers
    headers = {
        'User-Agent': user_agent,
        "host": "archive.is",
    }

    # Request a unique identifier for our activity
    logger.debug("Requesting {}".format(domain + "/"))
    response = requests.get(
        domain + "/",
        timeout=120,
        allow_redirects=True,
        headers=headers
    )

    # It will need to be parsed from the homepage response headers
    html = str(response.content)
    try:
        unique_id = html.split('name="submitid', 1)[1].split('value="', 1)[1].split('"', 1)[0]
        logger.debug("Unique identifier: {}".format(unique_id))
    except IndexError:
        logger.warn("Unable to extract unique identifier from archive.is. Submitting without it.")
        unique_id = None

    # Send the capture request to archive.is with the unique id included
    data = {
        "url": target_url,
        "anyway": 1,
    }
    if unique_id:
        data.update({"submitid": unique_id})

    logger.debug("Requesting {}".format(save_url))
    response = requests.post(
        save_url,
        timeout=120,
        allow_redirects=True,
        headers=headers,
        data=data
    )

    # archive.is returns a link format timemap in the header field link
    # but if it was the first time archive.is has archived the uri-r
    # or for some other reason unknown at this time
    # this information will not be present so resort to searching the
    # returned html page
    memento_re = re.compile('"(http(?:s)?://archive\.is/(?:[0-9]{14}/(?:\b)?)?'
                            '[-a-zA-Z0-9@:%_+.~#?&/=]+)"',
                            re.IGNORECASE | re.MULTILINE)
    mementos = memento_re.findall(response.text)

    logger.debug("Memento: {}".format(mementos[0]))

    # the url to the memento is the first element in the list
    return mementos[0]


@click.command()
@click.argument("url")
@click.option("-ua", "--user-agent", help="User-Agent header for the web request")
def cli(url, user_agent):
    """
    Archives the provided URL using archive.is.
    """
    kwargs = {}
    if user_agent:
        kwargs['user_agent'] = user_agent
    archive_url = capture(url, **kwargs)
    click.echo(archive_url)<commit_msg>Different parse routine<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import click
import logging
import requests
from six.moves.urllib.parse import urljoin
logger = logging.getLogger(__name__)


def capture(
    target_url,
    user_agent="archiveis (https://github.com/pastpages/archiveis)",
):
    """
    Archives the provided URL using archive.is

    Returns the URL where the capture is stored.
    """
    # Put together the URL that will save our request
    domain = "http://178.62.195.5"
    save_url = urljoin(domain, "/submit/")

    # Configure the request headers
    headers = {
        'User-Agent': user_agent,
        "host": "archive.is",
    }

    # Request a unique identifier for our activity
    logger.debug("Requesting {}".format(domain + "/"))
    response = requests.get(
        domain + "/",
        timeout=120,
        allow_redirects=True,
        headers=headers
    )

    # It will need to be parsed from the homepage response headers
    html = str(response.content)
    try:
        unique_id = html.split('name="submitid', 1)[1].split('value="', 1)[1].split('"', 1)[0]
        logger.debug("Unique identifier: {}".format(unique_id))
    except IndexError:
        logger.warn("Unable to extract unique identifier from archive.is. Submitting without it.")
        unique_id = None

    # Send the capture request to archive.is with the unique id included
    data = {
        "url": target_url,
        "anyway": 1,
    }
    if unique_id:
        data.update({"submitid": unique_id})

    logger.debug("Requesting {}".format(save_url))
    response = requests.post(
        save_url,
        timeout=120,
        allow_redirects=True,
        headers=headers,
        data=data
    )
    response.raise_for_status()

    # There are a couple ways the header can come back
    if 'Refresh' in response.headers:
        memento = str(response.headers['Refresh']).split(';url=')[1]
        logger.debug("Memento from Refresh header: {}".format(memento))
        return memento
    if 'Location' in response.headers:
        memento = response.headers['Location']
        logger.debug("Memento from Location header: {}".format(memento))
        return memento
    for i, r in enumerate(response.history):
        if 'Location' in r.headers:
            memento = r.headers['Location']
            logger.debug("Memento from the Location header of {} history response: {}".format(i+1, memento))
            return memento
    # If there's nothing at this point, throw an erro
    raise Exception("No memento returned by archive.is")


@click.command()
@click.argument("url")
@click.option("-ua", "--user-agent", help="User-Agent header for the web request")
def cli(url, user_agent):
    """
    Archives the provided URL using archive.is.
    """
    kwargs = {}
    if user_agent:
        kwargs['user_agent'] = user_agent
    archive_url = capture(url, **kwargs)
    click.echo(archive_url)
{'fraction_non_alphanumeric': 0.12710084033613445, 'fraction_numerical': 0.00980392156862745, 'mean_word_length': 2.704928664072633, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4252483', 'n_tokens_mistral': 1742, 'n_tokens_neox': 1686, 'n_words': 479}