<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
VMware vSphere driver using pyvmomi - https://github.com/vmware/pyvmomi
"""
import atexit
import collections
import os
import re
import ssl
import time
import functools
import warnings
from datetime import datetime
from datetime import timedelta

from libcloud.common.base import ConnectionUserAndKey
from libcloud.common.types import InvalidCredsError
from libcloud.common.types import LibcloudError
from libcloud.compute.base import Node
from libcloud.compute.base import NodeDriver
from libcloud.compute.base import NodeImage
from libcloud.compute.base import StorageVolume
from libcloud.compute.base import VolumeSnapshot
from libcloud.compute.types import NodeState
from libcloud.compute.types import Provider
from libcloud.utils import misc as misc_utils
from libcloud.utils import networking
from libcloud.utils.py3 import urlparse

try:
    from pyVim import connect
    from pyVmomi import vmodl
    from pyVmomi import vim
    from pyVmomi import vmodl
except ImportError:  # pragma: no cover
    raise ImportError(
        "Missing 'pyvmomi' dependency. You can install it "
        "using pip - 'pip install pyvmomi'")


__all__ = [
    'VSphereNodeDriver',
]


DEFAULT_CONNECTION_TIMEOUT = 5  # default connection timeout in seconds
DEFAULT_PAGE_SIZE = 1000

"""
VMWare vCenter Server stores events in the database for a limited period.
The default number of days to retain event messages in the database is 30.
"""
EVENTS_DAYS_LIMIT = 30


class VSpherePropertyCollector(misc_utils.PageList):
    # pylint: disable=line-too-long
    """
    The paginated PropertyCollector.

    See:
     - http://pubs.vmware.com/vsphere-50/index.jsp#com.vmware.wssdk.pg.doc_50/PG_Ch5_PropertyCollector.7.2.html
    """

    def __init__(
            self, driver, object_cls,
            container=None, path_set=None,
            process_fn=None, page_size=None):
        """
        :param driver: The VSphere driver.
        :type driver: :class:`VSphereNodeDriver`

        :param object_cls: Type of managed object.
        :type object_cls: :class:`vim.ManagedEntity`

        :param container: The :class:`vim.Folder` or :class:`vim.Datacenter`
            instance that provides the objects that the view presents.
        :type container: :class:`vim.Folder` or :class:`vim.Datacenter`

        :param path_set: List of properties to retrieve, defaults to ``None``.
        :type path_set: list or None

        :param page_size: The size of the result page,
            defaults to ``DEFAULT_PAGE_SIZE``.
        :param page_size: int or None

        :param process_fn: Function to be applied to the page result,
            defaults to ``None``.
        :param process_fn: callable or None
        """
        self._connection = driver.connection
        self._object_cls = object_cls
        self._path_set = path_set

        self._container = container
        self._container_view = None
        self._collector = None

        def _process_fn(result):
            """Convert result to list|dict before processing."""
            if result is None:
                raise LibcloudError((
                    "VSpherePropertyCollector cannot retrieve {} objects"
                ).format(self._object_cls.__name__))
            if path_set is None:
                result = [prop.obj for prop in result.objects]
            else:
                result = {
                    prop.obj: {prop.name: prop.val for prop in prop.propSet}
                    for prop in result.objects}
            if isinstance(result, dict) and process_fn is None:
                raise LibcloudError(
                    "VSpherePropertyCollector: 'process_fn' parameter "
                    "must be specified.")
            return process_fn(result) if process_fn else result

        super(VSpherePropertyCollector, self).__init__(
            request_fn=self._retrieve_properties,
            process_fn=_process_fn,
            page_size=page_size or DEFAULT_PAGE_SIZE,
            request_args=[],
            request_kwargs={})

    def extract_next_page_token(self, response):
        """
        :type response: :class:`vmodl.query.PropertyCollector.RetrieveResult`
        """
        return response.token

    def _retrieve_properties(self):
        if self.next_page_token is None:<commit_msg>SCALRCORE-9672 fix pagination for empty response<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
VMware vSphere driver using pyvmomi - https://github.com/vmware/pyvmomi
"""
import atexit
import collections
import os
import re
import ssl
import time
import functools
import warnings
from datetime import datetime
from datetime import timedelta

from libcloud.common.base import ConnectionUserAndKey
from libcloud.common.types import InvalidCredsError
from libcloud.common.types import LibcloudError
from libcloud.compute.base import Node
from libcloud.compute.base import NodeDriver
from libcloud.compute.base import NodeImage
from libcloud.compute.base import StorageVolume
from libcloud.compute.base import VolumeSnapshot
from libcloud.compute.types import NodeState
from libcloud.compute.types import Provider
from libcloud.utils import misc as misc_utils
from libcloud.utils import networking
from libcloud.utils.py3 import urlparse

try:
    from pyVim import connect
    from pyVmomi import vmodl
    from pyVmomi import vim
    from pyVmomi import vmodl
except ImportError:  # pragma: no cover
    raise ImportError(
        "Missing 'pyvmomi' dependency. You can install it "
        "using pip - 'pip install pyvmomi'")


__all__ = [
    'VSphereNodeDriver',
]


DEFAULT_CONNECTION_TIMEOUT = 5  # default connection timeout in seconds
DEFAULT_PAGE_SIZE = 1000

"""
VMWare vCenter Server stores events in the database for a limited period.
The default number of days to retain event messages in the database is 30.
"""
EVENTS_DAYS_LIMIT = 30


class VSpherePropertyCollector(misc_utils.PageList):
    # pylint: disable=line-too-long
    """
    The paginated PropertyCollector.

    See:
     - http://pubs.vmware.com/vsphere-50/index.jsp#com.vmware.wssdk.pg.doc_50/PG_Ch5_PropertyCollector.7.2.html
    """

    def __init__(
            self, driver, object_cls,
            container=None, path_set=None,
            process_fn=None, page_size=None):
        """
        :param driver: The VSphere driver.
        :type driver: :class:`VSphereNodeDriver`

        :param object_cls: Type of managed object.
        :type object_cls: :class:`vim.ManagedEntity`

        :param container: The :class:`vim.Folder` or :class:`vim.Datacenter`
            instance that provides the objects that the view presents.
        :type container: :class:`vim.Folder` or :class:`vim.Datacenter`

        :param path_set: List of properties to retrieve, defaults to ``None``.
        :type path_set: list or None

        :param page_size: The size of the result page,
            defaults to ``DEFAULT_PAGE_SIZE``.
        :param page_size: int or None

        :param process_fn: Function to be applied to the page result,
            defaults to ``None``.
        :param process_fn: callable or None
        """
        self._connection = driver.connection
        self._object_cls = object_cls
        self._path_set = path_set

        self._container = container
        self._container_view = None
        self._collector = None

        def _process_fn(result):
            """Convert result to list|dict before processing."""
            if result is None:
                return []
            if path_set is None:
                result = [prop.obj for prop in result.objects]
            else:
                result = {
                    prop.obj: {prop.name: prop.val for prop in prop.propSet}
                    for prop in result.objects}
            if isinstance(result, dict) and process_fn is None:
                raise LibcloudError(
                    "VSpherePropertyCollector: 'process_fn' parameter "
                    "must be specified.")
            return process_fn(result) if process_fn else result

        super(VSpherePropertyCollector, self).__init__(
            request_fn=self._retrieve_properties,
            process_fn=_process_fn,
            page_size=page_size or DEFAULT_PAGE_SIZE,
            request_args=[],
            request_kwargs={})

    def extract_next_page_token(self, response):
        """
        :type response: :class:`vmodl.query.PropertyCollector.RetrieveResult`
        """
        return response.token if response else None

    def _retrieve_properties(self):
        if self.next_page_token is None:
{'fraction_non_alphanumeric': 0.07072303437376531, 'fraction_numerical': 0.004543658632951403, 'mean_word_length': 2.846884498480243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325390', 'n_tokens_mistral': 2753, 'n_tokens_neox': 2667, 'n_words': 951}