<commit_before>
import os
import json
from xml.etree import ElementTree
import requests
from requests_futures.sessions import FuturesSession

from usgs import USGS_API, USGSError
from usgs import soap, xsi


TMPFILE = os.path.join("/", "tmp", "usgs")
NAMESPACES = {
    "SOAP-ENV": "http://schemas.xmlsoap.org/soap/envelope/",
    "ns1": "https://earthexplorer.usgs.gov/inventory/soap",
    "eemetadata": "http://earthexplorer.usgs.gov/eemetadata.xsd"
}

def _get_api_key():

    api_key = None

    if os.path.exists(TMPFILE):
        with open(TMPFILE, "r") as f:
            api_key = f.read()

    return api_key


def _check_for_usgs_error(root):

    fault_code_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultcode", NAMESPACES)

    if fault_code_el is None:
        return

    fault_string_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", NAMESPACES)

    fault_code = fault_code_el.text
    fault_string = fault_string_el.text

    raise USGSError('%s: %s' % (fault_code, fault_string))


def _get_extended(scene, resp):
    """
    Parse metadata returned from the metadataUrl of a USGS scene.

    :param scene:
        Dictionary representation of a USGS scene
    :param resp:
        Response object from requests/grequests
    """
    root = ElementTree.fromstring(resp.text)
    items = root.findall("eemetadata:metadataFields/eemetadata:metadataField", NAMESPACES)
    scene['extended'] = { item.attrib.get('name').strip(): xsi.get(item[0]) for item in items }

    return scene


def _async_requests(urls):
    """
    Sends multiple non-blocking requests. Returns
    a list of responses.

    :param urls:
        List of urls
    """

    session = FuturesSession(max_workers=30)
    futures = [ session.get(url) for url in urls ]
    return [ future.result() for future in futures ]


def _get_metadata_url(scene):
    return scene.get('metadataUrl')


def clear_bulk_download_order():
    raise NotImplementedError


def clear_order():
    raise NotImplementedError


def datasets(dataset, node, ll=None, ur=None, start_date=None, end_date=None):

    api_key = _get_api_key()

    xml = soap.datasets(dataset, node, ll=ll, ur=ur, start_date=start_date, end_date=end_date, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetsResponse/return/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data


def dataset_fields(dataset, node):

    api_key = _get_api_key()

    xml = soap.dataset_fields(dataset, node, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetFieldsResponse/return/item", NAMESPACES)
    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data


def download(dataset, node, entityids, product='STANDARD', api_key=None):
    """
    Though USGS supports multiple products in a single request, there's
    ambiguity in the returned list. This wrapper only allows a single
    product per request.

    Additionally, the response has no indiction which URL is associated
    with which scene/entity id. The URL can be parsed, but the structure
    varies depending on the product.
    """

    api_key = api_key if api_key else _get_api_key()

    xml = soap.download(dataset, node, entityids, [product], api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadResponse/return/item", NAMESPACES)

    data = map(lambda el: xsi.get(el), items)

    return data


def download_options(dataset, node, entityids):

    api_key = _get_api_key()

    xml = soap.download_options(dataset, node, entityids, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadOptionsResponse/return/item/downloadOptions/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data


def get_bulk_download_products():
    raise NotImplementedError


def get_order_products():
    raise NotImplementedError


def hits():
    raise NotImplementedError


def item_basket():
    raise NotImplementedError


def login(username, password):
    xml = soap.login(username, password)
    r = requests.post(USGS_API, xml)

    if r.status_code is not 200:
        raise USGSError(r.text)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    element = root.find("SOAP-ENV:Body/ns1:loginResponse/return", NAMESPACES)

    api_key = element.text

    with open(TMPFILE, "w") as f:
        f.write(api_key)

    return api_key


def logout():

    api_key = _get_api_key()
<commit_msg>save api key to disk if requested<commit_after>
import os
import json
from xml.etree import ElementTree
import requests
from requests_futures.sessions import FuturesSession

from usgs import USGS_API, USGSError
from usgs import soap, xsi


TMPFILE = os.path.join("/", "tmp", "usgs")
NAMESPACES = {
    "SOAP-ENV": "http://schemas.xmlsoap.org/soap/envelope/",
    "ns1": "https://earthexplorer.usgs.gov/inventory/soap",
    "eemetadata": "http://earthexplorer.usgs.gov/eemetadata.xsd"
}

def _get_api_key():

    api_key = None

    if os.path.exists(TMPFILE):
        with open(TMPFILE, "r") as f:
            api_key = f.read()

    return api_key


def _check_for_usgs_error(root):

    fault_code_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultcode", NAMESPACES)

    if fault_code_el is None:
        return

    fault_string_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", NAMESPACES)

    fault_code = fault_code_el.text
    fault_string = fault_string_el.text

    raise USGSError('%s: %s' % (fault_code, fault_string))


def _get_extended(scene, resp):
    """
    Parse metadata returned from the metadataUrl of a USGS scene.

    :param scene:
        Dictionary representation of a USGS scene
    :param resp:
        Response object from requests/grequests
    """
    root = ElementTree.fromstring(resp.text)
    items = root.findall("eemetadata:metadataFields/eemetadata:metadataField", NAMESPACES)
    scene['extended'] = { item.attrib.get('name').strip(): xsi.get(item[0]) for item in items }

    return scene


def _async_requests(urls):
    """
    Sends multiple non-blocking requests. Returns
    a list of responses.

    :param urls:
        List of urls
    """

    session = FuturesSession(max_workers=30)
    futures = [ session.get(url) for url in urls ]
    return [ future.result() for future in futures ]


def _get_metadata_url(scene):
    return scene.get('metadataUrl')


def clear_bulk_download_order():
    raise NotImplementedError


def clear_order():
    raise NotImplementedError


def datasets(dataset, node, ll=None, ur=None, start_date=None, end_date=None):

    api_key = _get_api_key()

    xml = soap.datasets(dataset, node, ll=ll, ur=ur, start_date=start_date, end_date=end_date, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetsResponse/return/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data


def dataset_fields(dataset, node):

    api_key = _get_api_key()

    xml = soap.dataset_fields(dataset, node, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetFieldsResponse/return/item", NAMESPACES)
    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data


def download(dataset, node, entityids, product='STANDARD', api_key=None):
    """
    Though USGS supports multiple products in a single request, there's
    ambiguity in the returned list. This wrapper only allows a single
    product per request.

    Additionally, the response has no indiction which URL is associated
    with which scene/entity id. The URL can be parsed, but the structure
    varies depending on the product.
    """

    api_key = api_key if api_key else _get_api_key()

    xml = soap.download(dataset, node, entityids, [product], api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadResponse/return/item", NAMESPACES)

    data = map(lambda el: xsi.get(el), items)

    return data


def download_options(dataset, node, entityids):

    api_key = _get_api_key()

    xml = soap.download_options(dataset, node, entityids, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadOptionsResponse/return/item/downloadOptions/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data


def get_bulk_download_products():
    raise NotImplementedError


def get_order_products():
    raise NotImplementedError


def hits():
    raise NotImplementedError


def item_basket():
    raise NotImplementedError


def login(username, password, save=False):
    xml = soap.login(username, password)
    r = requests.post(USGS_API, xml)

    if r.status_code is not 200:
        raise USGSError(r.text)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    element = root.find("SOAP-ENV:Body/ns1:loginResponse/return", NAMESPACES)

    api_key = element.text

    if save:
        with open(TMPFILE, "w") as f:
            f.write(api_key)

    return api_key


def logout():

    api_key = _get_api_key()

{'fraction_non_alphanumeric': 0.1076031571585573, 'fraction_numerical': 0.002397841942251973, 'mean_word_length': 3.647168059424327, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1242347', 'n_tokens_mistral': 3510, 'n_tokens_neox': 3375, 'n_words': 795}