<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Packet Driver
"""
import multiprocessing.pool

from libcloud.utils.py3 import httplib

from libcloud.common.base import ConnectionKey, JsonResponse
from libcloud.compute.types import Provider, NodeState, InvalidCredsError
from libcloud.compute.base import NodeDriver, Node
from libcloud.compute.providers import get_driver
from libcloud.compute.base import NodeImage, NodeSize, NodeLocation
from libcloud.compute.base import KeyPair

PACKET_ENDPOINT = "api.packet.net"


class PacketResponse(JsonResponse):
    valid_response_codes = [httplib.OK, httplib.ACCEPTED, httplib.CREATED,
                            httplib.NO_CONTENT]

    def parse_error(self):
        if self.status == httplib.UNAUTHORIZED:
            body = self.parse_body()
            raise InvalidCredsError(body.get('error'))
        else:
            body = self.parse_body()
            if 'message' in body:
                error = '%s (code: %s)' % (body.get('message'), self.status)
            elif 'errors' in body:
                error = body.get('errors')
            else:
                error = body
            raise Exception(error)

    def success(self):
        return self.status in self.valid_response_codes


class PacketConnection(ConnectionKey):
    """
    Connection class for the Packet driver.
    """

    host = PACKET_ENDPOINT
    responseCls = PacketResponse

    def add_default_headers(self, headers):
        """
        Add headers that are necessary for every request
        """
        headers['Content-Type'] = 'application/json'
        headers['X-Auth-Token'] = self.key
        headers['X-Consumer-Token'] = \
            '<KEY>'
        return headers


class PacketNodeDriver(NodeDriver):
    """
    Packet NodeDriver
    """

    connectionCls = PacketConnection
    type = Provider.PACKET
    name = 'Packet'
    website = 'http://www.packet.net/'

    NODE_STATE_MAP = {'queued': NodeState.PENDING,
                      'provisioning': NodeState.PENDING,
                      'rebuilding': NodeState.PENDING,
                      'powering_on': NodeState.REBOOTING,
                      'powering_off': NodeState.REBOOTING,
                      'rebooting': NodeState.REBOOTING,
                      'inactive': NodeState.STOPPED,
                      'deleted': NodeState.TERMINATED,
                      'deprovisioning': NodeState.TERMINATED,
                      'failed': NodeState.ERROR,
                      'active': NodeState.RUNNING}

    def __init__(self, key, project=None):
        # initialize a NodeDriver for packet.net using the API token
        # and optionally the project (name or id)
        # If project specified we need to be sure this is a valid project
        # so we create the variable self.project_id
        super(PacketNodeDriver, self).__init__(key=key, project=None)
        self.project_name = project
        self.project_id = None
        self.projects = self.ex_list_projects()
        if project:
            for project_obj in self.projects:
                if project in [project_obj.name, project_obj.id]:
                    self.project_id = project_obj.id
                    break
            if not self.project_id:
                self.project_name = None

    def ex_list_projects(self):
        projects = []
        data = self.connection.request('/projects').object
        projects = data.get('projects')
        if projects:
            projects = [Project(project) for project in projects]
        return projects

    def list_nodes(self, ex_project_id=None):
        if ex_project_id:
            return self.list_nodes_for_project(ex_project_id=ex_project_id)
        else:
            # if project has been specified on initialization of driver, then
            # return nodes for this project only
            if self.project_id:
                return self.list_nodes_for_project(
                    ex_project_id=self.project_id)
            else:
                projects = [project.id for project in self.projects]

                def _list_one(project):
                    driver = get_driver(self.type)(self.key)
                    try:
                        return driver.list_nodes_for_project(project)
                    except Exception:
                        return []
                pool = multiprocessing.pool.ThreadPool(8)
                results = pool.map(_list_one, projects)
                pool.terminate()
                nodes = []
                for result in results:
                    nodes.extend(result)
                return nodes

    def list_nodes_for_project(self, ex_project_id, include='plan', page=1,
                               per_page=1000):
        params = {
            'include': include,
            'page': page,
            'per_page': per_page
        }
        data = self.connection.request(
            '/projects/%s/devices' % (ex_project_id),
            params=params).object['devices']
        return list(map(self._to_node, data))

    def list_locations(self):
        data = self.connection.request('/facilities')\<commit_msg>Use asyncio instead of multiprocessing to parallelize requests in Packet driver in case of Python3, while maintaining Python2 compatibility<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Packet Driver
"""
try:
    import asyncio
except ImportError:
    asyncio = None

from libcloud.utils.py3 import httplib

from libcloud.common.base import ConnectionKey, JsonResponse
from libcloud.compute.types import Provider, NodeState, InvalidCredsError
from libcloud.compute.base import NodeDriver, Node
from libcloud.compute.providers import get_driver
from libcloud.compute.base import NodeImage, NodeSize, NodeLocation
from libcloud.compute.base import KeyPair

PACKET_ENDPOINT = "api.packet.net"


class PacketResponse(JsonResponse):
    valid_response_codes = [httplib.OK, httplib.ACCEPTED, httplib.CREATED,
                            httplib.NO_CONTENT]

    def parse_error(self):
        if self.status == httplib.UNAUTHORIZED:
            body = self.parse_body()
            raise InvalidCredsError(body.get('error'))
        else:
            body = self.parse_body()
            if 'message' in body:
                error = '%s (code: %s)' % (body.get('message'), self.status)
            elif 'errors' in body:
                error = body.get('errors')
            else:
                error = body
            raise Exception(error)

    def success(self):
        return self.status in self.valid_response_codes


class PacketConnection(ConnectionKey):
    """
    Connection class for the Packet driver.
    """

    host = PACKET_ENDPOINT
    responseCls = PacketResponse

    def add_default_headers(self, headers):
        """
        Add headers that are necessary for every request
        """
        headers['Content-Type'] = 'application/json'
        headers['X-Auth-Token'] = self.key
        headers['X-Consumer-Token'] = \
            '<KEY>'
        return headers


class PacketNodeDriver(NodeDriver):
    """
    Packet NodeDriver
    """

    connectionCls = PacketConnection
    type = Provider.PACKET
    name = 'Packet'
    website = 'http://www.packet.com/'

    NODE_STATE_MAP = {'queued': NodeState.PENDING,
                      'provisioning': NodeState.PENDING,
                      'rebuilding': NodeState.PENDING,
                      'powering_on': NodeState.REBOOTING,
                      'powering_off': NodeState.REBOOTING,
                      'rebooting': NodeState.REBOOTING,
                      'inactive': NodeState.STOPPED,
                      'deleted': NodeState.TERMINATED,
                      'deprovisioning': NodeState.TERMINATED,
                      'failed': NodeState.ERROR,
                      'active': NodeState.RUNNING}

    def __init__(self, key, project=None):
        # initialize a NodeDriver for Packet using the API token
        # and optionally the project (name or id)
        # If project specified we need to be sure this is a valid project
        # so we create the variable self.project_id
        super(PacketNodeDriver, self).__init__(key=key, project=None)
        self.project_name = project
        self.project_id = None
        self.projects = self.ex_list_projects()
        if project:
            for project_obj in self.projects:
                if project in [project_obj.name, project_obj.id]:
                    self.project_id = project_obj.id
                    break
            if not self.project_id:
                self.project_name = None

    def ex_list_projects(self):
        projects = []
        data = self.connection.request('/projects').object
        projects = data.get('projects')
        if projects:
            projects = [Project(project) for project in projects]
        return projects

    def list_nodes(self, ex_project_id=None):
        if ex_project_id:
            return self.list_nodes_for_project(ex_project_id=ex_project_id)
        else:
            # if project has been specified during driver initialization, then
            # return nodes for this project only
            if self.project_id:
                return self.list_nodes_for_project(
                    ex_project_id=self.project_id)

            # In case of Python2 perform requests serially
            if asyncio == None:
                nodes = []
                for project in self.projects:
                    nodes.extend(
                        self.list_nodes_for_project(ex_project_id=project.id)
                    )
                return nodes

            # In case of Python3 use asyncio to perform requests in parallel
            # The _list_nodes function is defined dynamically using exec in
            # order to prevent a SyntaxError in Python2 due to "yield from".
            # This cruft can be removed once Python2 support is no longer
            # required.
            glob = globals()
            loc = locals()
            exec("""
import asyncio
@asyncio.coroutine
def _list_nodes(driver):
    projects = [project.id for project in driver.projects]
    loop = asyncio.get_event_loop()
    futures = [
        loop.run_in_executor(None, driver.list_nodes_for_project, p)
        for p in projects
    ]
    nodes = []
    for future in futures:
        result = yield from future
        nodes.extend(result)
    return nodes""", glob , loc)
            loop = asyncio.get_event_loop()
            nodes = loop.run_until_complete(loc['_list_nodes'](loc['self']))
            return nodes

    def list_nodes_for_project(self, ex_project_id, include='plan', page=1,
                               per_page=1000):
        params = {
            'include': include,
            'page': page,
            'per_page': per_page
        }
        data = self.connection.request(
            '/projects/%s/devices' % (ex_project_id),
            params=params).object['devices']
        return list(map(self._to_node, data))

    def list_locations(self):
        data = self.connection.request('/facilities')\
{'fraction_non_alphanumeric': 0.08164561988722104, 'fraction_numerical': 0.002144388849177984, 'mean_word_length': 2.1191478820906613, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1538069', 'n_tokens_mistral': 3335, 'n_tokens_neox': 3216, 'n_words': 966}