<commit_before># -*- coding: utf-8  -*-
"""
Functions for manipulating external links
or querying third-party sites.

"""
#
# (C) Pywikibot team, 2013
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'

import pywikibot
from pywikibot.comms import http


def getInternetArchiveURL(url, timestamp=None):
    """Return archived URL by Internet Archive."""
    # See [[:mw:Archived Pages]] and http://archive.org/help/wayback_api.php
    import json
    query = u'http://archive.org/wayback/available?'
    query += u'url='
    query += url
    if not timestamp is None:
        query += u'&timestamp='
        query += timestamp
    jsontext = http.request(uri=query, site=None)
    if "closest" in jsontext:
        data = json.loads(jsontext)
        return data['archived_snapshots']['closest']['url']
    else:
        return None


def getWebCitationURL(url, timestamp=None):
    """Return archived URL by Web Citation."""
    # See http://www.webcitation.org/doc/WebCiteBestPracticesGuide.pdf
    import xml.etree.ElementTree as ET
    query = u'http://www.webcitation.org/query?'
    query += u'returnxml=true'
    query += u'&url='
    query += url
    if not timestamp is None:
        query += u'&date='
        query += timestamp
    xmltext = http.request(uri=query, site=None)
    if "success" in xmltext:
        data = ET.fromstring(xmltext)
        return data.find('.//webcite_url').text
    else:
        return None<commit_msg>Weblib: add docs, replace string concat with urlencode<commit_after># -*- coding: utf-8  -*-
"""
Functions for manipulating external links
or querying third-party sites.

"""
#
# (C) Pywikibot team, 2013
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'

import urllib
from pywikibot.comms import http


def getInternetArchiveURL(url, timestamp=None):
    """Return archived URL by Internet Archive.

    Parameters:
        url - url to search an archived version for
        timestamp - requested archive date. The version closest to that moment
                    is returned. Format: YYYYMMDDhhmmss or part thereof.

    See [[:mw:Archived Pages]] and http://archive.org/help/wayback_api.php
    for more details.
    """
    import json
    uri = u'http://archive.org/wayback/available?'

    query = {'url': url}

    if timestamp is not None:
        query['timestamp'] = timestamp

    uri = uri + urllib.urlencode(query)
    jsontext = http.request(uri=uri, site=None)
    if "closest" in jsontext:
        data = json.loads(jsontext)
        return data['archived_snapshots']['closest']['url']
    else:
        return None


def getWebCitationURL(url, timestamp=None):
    """Return archived URL by Web Citation.

    Parameters:
        url - url to search an archived version for
        timestamp - requested archive date. The version closest to that moment
                    is returned. Format: YYYYMMDDhhmmss or part thereof.

    See http://www.webcitation.org/doc/WebCiteBestPracticesGuide.pdf
    for more details
    """
    import xml.etree.ElementTree as ET
    uri = u'http://www.webcitation.org/query?'

    query = {'returnxml': 'true',
             'url': url}

    if not timestamp is None:
        query['date'] = timestamp

    uri = uri + urllib.urlencode(query)
    xmltext = http.request(uri=uri, site=None)
    if "success" in xmltext:
        data = ET.fromstring(xmltext)
        return data.find('.//webcite_url').text
    else:
        return None
{'fraction_non_alphanumeric': 0.11577133007756392, 'fraction_numerical': 0.002872737719046251, 'mean_word_length': 2.822173435784852, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2928606', 'n_tokens_mistral': 1087, 'n_tokens_neox': 1040, 'n_words': 310}