<commit_before># Copyright (c) 2014 <NAME>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Apache Libcloud compute driver implementation for Vagrant."""

import itertools
import logging
import os
import pwd
import re
import subprocess
import time

from libcloud.common.types import LibcloudError
from libcloud.compute import base
from libcloud.compute.types import NodeState

from libcloudvagrant import virtualbox
from libcloudvagrant.catalogue import VagrantCatalogue
from libcloudvagrant.types import (
    VagrantImage,
    VagrantNetwork,
    VagrantNode,
    VagrantNodeSize,
    VagrantVolume,
)


__all__ = [
    "VAGRANT",
    "VagrantDriver",
]


VAGRANT = "vagrant"

_HOME = pwd.getpwuid(os.getuid()).pw_dir


class VagrantDriver(base.NodeDriver):

    """Apache Libcloud driver implementation.

    Docstrings here document how this driver diverges from the base class.

    """

    type = VAGRANT
    name = "Vagrant"
    website = "http://www.vagrantup.com/"

    log = logging.getLogger("libcloudvagrant")

    def __init__(self):
        super(VagrantDriver, self).__init__(key=None)

    def attach_volume(self, node, volume, device=None):
        """Attaches volume to node.

        At the moment only SATA devices of the form ``/dev/sd[a-z]`` are
        accepted as values to parameter ``device``.

        :param node: Node to attach volume to.
        :type node: :class:`VagrantNode`

        :param volume: Volume to attach.
        :type volume: :class:`VagrantVolume`

        :param device: Where the device is exposed, e.g. '/dev/sdb'
        :type device: ``str``

        :rytpe: ``bool``

        """
        if volume.attached_to:
            self.log.warn("Volume %s already attached to %s",
                          volume.name, volume.attached_to)
            return False

        if node.state not in (NodeState.STOPPED, NodeState.UNKNOWN):
            self.log.warn("Cannot attach volumes to running nodes")
            return False

        try:
            with self._catalogue as c:
                node_uuid = c.virtualbox_uuid(node)
                virtualbox.attach_volume(node_uuid, volume.path, device)
                volume.attached_to = node.name
                c.update_volume(volume)
        except:
            self.log.warn("Error attaching %s to %s", volume, node,
                          exc_info=True)
            return False
        self.log.info("Volume '%s' attached to node '%s'",
                      volume.name, node.name)
        return True

    def create_node(self, name, size, image, ex_networks=None, **kwargs):
        """Create a new node instance. This instance will be started
        automatically.

        Requires the following arguments:

        :param name:   String with a name for this new node
        :type name:   ``str``

        :param size:   The size of resources allocated to this node.
        :type size:   :class:`VagrantSize`

        :param image:  OS Image to boot on node.
        :type image:  :class:`VagrantImage`

        Accepts the following non-standard arguments:

        :param ex_networks: The networks to connect this node to.
        :type ex_networks:  ``list`` of :class:`VagrantNetwork`

        All other arguments are ignored.

        """
        if ex_networks is None:
            networks = []
        else:
            networks = ex_networks

        self.log.info("Creating node '%s' ..", name)

        with self._catalogue as c:
            public_ips = [n.allocate_address().to_dict()
                          for n in networks if n.public]
            private_ips = [n.allocate_address().to_dict()
                           for n in networks if not n.public]
            size = size.to_dict()
            image = image.to_dict()
            for n in networks:
                c.update_network(n)
            node = VagrantNode(name=name,
                               public_ips=public_ips,
                               private_ips=private_ips,
                               driver=self,
                               size=size,
                               image=image)
            self.log.debug("create_node(%s): Created object: %s", name, node)
            c.add_node(node)
            c.save()  # Explicit save, so that the next command succeeds
            self.ex_start_node(node)
            self.log.info(".. Node '%s' created", name)
            return node

    def create_volume(self, size, name, **kwargs):
        """Create a new volume.

        :param size: Size of volume in gigabytes (required)
        :type size: ``int``

        :param name: Name of the volume to be created
        :type name: ``str``

        All other arguments are ignored.

        :return: The newly created volume.
        :rtype: :class:`VagrantVolume`

        """
        with self._catalogue as c:
            path = c.volume_path("%s.vdi" % (name,))
            virtualbox.create_volume(path=path, size=size * 1024)
            volume = VagrantVolume(name=name,
                                   size=size,
                                   extra={
                                       "attached_to": None,
                                       "path": path,
                                   },
                                   driver=self)
            c.add_volume(volume)
            self.log.info("Volume '%s' created", name)
            return volume

    def delete_image(self, image):
        """Deletes a node image from a provider.

        :param image: Node image object.
        :type image:  :class:`VagrantImage`

        :return: ``True`` if delete_image was successful, ``False`` otherwise.
        :rtype: ``bool``

        """
        try:
            self._vagrant("box remove --force --provider virtualbox", image.id)
            return True
        except:
            self.log.warn("Cannot remove image %s", image, exc_info=True)
            return False

    def deploy_node(self, **kwargs):
        """Create a new node, and start deployment.

        This method calls ``Node.deploy_node()`` with the default Vagrant SSH
        connection parameters and credentials.

        """
        ssh_config = self._vagrant_ssh_config()
        kwargs["ssh_username"] = ssh_config["user"]
        kwargs["ssh_port"] = ssh_config["port"]
        kwargs["ssh_key"] = ssh_config["key"]
        kwargs["ssh_interface"] = ssh_config["host"]
        self.log.debug("Deploy args: %s", kwargs)
        super(VagrantDriver, self).deploy_node(**kwargs)

    def detach_volume(self, volume):
        """Detaches a volume from a node.

        :param volume: Volume to be detached
        :type volume: :class:`VagrantVolume`

        :rtype: ``bool`

        """
        node = volume.attached_to
        if not node:
            return True
        try:
            with self._catalogue as c:
                node_uuid = c.virtualbox_uuid(node)
                virtualbox.detach_volume(node_uuid, volume.path)
                volume.attached_to = None
                c.update_volume(volume)
                return True
        except:
            self.log.warn("Cannot detach volume %s", volume.name,
                          exc_info=True)
            return False

    def destroy_node(self, node):
        """Destroy a node.

        Volumes attached to this node and networks this node is connected to
        are not destroyed.

        :param node: The node to be destroyed
        :type node: :class:`VagrantNode`

        :return: True if the destroy was successful, False otherwise.
        :rtype: ``bool``

        """
        try:
            with self._catalogue as c:
                self._vagrant("destroy --force", node.name)
                for ip in node._public_ips + node._private_ips:
                    self.log.debug("destroy_node(): Deallocating address %s",
                                   ip)
                    n = c.find_network(ip.network_name)
                    n.deallocate_address(ip.address)
                    c.update_network(n)
                for v in c.get_volumes():
                    if v.attached_to == node.name:
                        self.log.debug("destroy_node(): Detaching %s", v)
                        self.detach_volume(v)
                c.remove_node(node)
            return True
        except:
            self.log.warn("Cannot destroy %s", node.name, exc_info=True)
            return False

    def destroy_volume(self, volume):
        """Destroys a storage volume.

        :param volume: Volume to be destroyed
        :type volume: :class:`VagrantVolume`

        :return: True if the destroy was successful, False otherwise.
        :rtype: ``bool``

        """
        if volume.attached_to:
            self.log.warn("Cannot destroy volume %s: It is attached to %s",
                          volume.name, volume.attached_to)
            return False
        with self._catalogue as c:
            c.remove_volume(volume)
            try:
                os.unlink(volume.path)
            except IOError:
                self.log.warn("Cannot unlink %s", volume.path, exc_info=True)
        return True

    def get_image(self, image_id):
        """Returns a Vagrant image object.

        :param image_id: Image to retrieve (like ``hashicorp/precise64``).
        :type image_id: ``str``

        :return: Image instance on success.
        :rtype :class:`VagrantImage`

        """
        def find_image():
            for image in self.list_images():
                if image.id == image_id:
                    return image

        image = find_image()
        if image:
            return image

        self.log.info("Fetching image '%s' ..", image_id)
        self._vagrant("box add --provider virtualbox", image_id)
        self.log.info(".. Done fetching image '%s'", image_id)

        return find_image()

    def list_images(self, location=None):
        """Lists registered images

        :return: A list of registered images
        :rtype: ``list`` of :class:`VagrantImage`

        """
        images = []
        cur = None
        for line in self._vagrant("box list").strip().split("\n"):
            self.log.debug("Scanning [%s]", line)
            if not line:
                continue
            bits = line.split(",")
            key = bits[2]
            data = bits[3]
            if key == "box-name":
                if cur is not None:
                    images.append(cur)
                cur = {"box-name": data}
            elif key == "box-provider":
                cur["box-provider"] = data
        if cur:
            images.append(cur)

        return [VagrantImage(name=i["box-name"], driver=self)
                for i in images if i["box-provider"] == "virtualbox"]

    def list_nodes(self):
        """Lists all registered nodes.

        :return:  A list of node objects
        :rtype: ``list`` of :class:`VagrantNode`

        """
        with self._catalogue as catalogue:
            nodes = catalogue.get_nodes()
            self.log.debug("Catalogue nodes: %s", nodes)
            return nodes

    def list_sizes(self, location=None):
        """Returns the single size object defined.

        The default size object instructs ``libcloud`` to create a node with
        the same amount of memory and number of CPUs as those of the Vagrant
        image it is created from.

        The ``location`` argument is ignored.

        :return: A list of one single size object
        :rtype: ``list`` of :class:`VagrantSize`

        """
        return [VagrantNodeSize(name="default",
                                ram=0,
                                driver=self,
                                extra={"cpu": 0})]

    def list_volumes(self):
        """Lists all registered storage volumes.

        :rtype: ``list`` of :class:`VagrantVolume`

        """
        with self._catalogue as c:
            ret = c.get_volumes()
            self.log.debug("list_volumes(): Returning %s", ret)
            return ret

    def reboot_node(self, node):
        """Reboot a node.

        :param node: The node to be rebooted
        :type node: :class:`VagrantNode`

        :return: ``True`` if the reboot was successful, otherwise ``False``
        :rtype: ``bool``

        """
        try:
            self._vagrant("reload --no-provision", node.name)
            return True
        except:
            self.log.debug("Cannot reload %s", node.name, exc_info=True)

    def wait_until_running(self, nodes, wait_period=3, timeout=600,
                           ssh_interface="public_ips", force_ipv4=True):
        """Block until the provided nodes are considered running.

        Unlike its overridden version, this method does not require any public
        or private IP address to be assigned to the host, since Vagrant's NAT
        interface is always available when a node's state is
        ``NodeState.RUNNING``.

        :param nodes: List of nodes to wait for.
        :type nodes: ``list`` of :class:`VagrantNode`

        :param wait_period: How many seconds to wait between each loop
                            iteration. (default is 3)
        :type wait_period: ``int``

        :param timeout: How many seconds to wait before giving up.
                        (default is 600)
        :type timeout: ``int``

        :param ssh_interface: Ignored parameter.
        :type ssh_interface: ``str``

        :param force_ipv4: Ignored parameter
        :type force_ipv4: ``bool``

        :return: ``[(VagrantNode, ip_addresses)]`` list of tuple of<commit_msg>Ve more verbose when rebooting a node<commit_after># Copyright (c) 2014 <NAME>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Apache Libcloud compute driver implementation for Vagrant."""

import itertools
import logging
import os
import pwd
import re
import subprocess
import time

from libcloud.common.types import LibcloudError
from libcloud.compute import base
from libcloud.compute.types import NodeState

from libcloudvagrant import virtualbox
from libcloudvagrant.catalogue import VagrantCatalogue
from libcloudvagrant.types import (
    VagrantImage,
    VagrantNetwork,
    VagrantNode,
    VagrantNodeSize,
    VagrantVolume,
)


__all__ = [
    "VAGRANT",
    "VagrantDriver",
]


VAGRANT = "vagrant"

_HOME = pwd.getpwuid(os.getuid()).pw_dir


class VagrantDriver(base.NodeDriver):

    """Apache Libcloud driver implementation.

    Docstrings here document how this driver diverges from the base class.

    """

    type = VAGRANT
    name = "Vagrant"
    website = "http://www.vagrantup.com/"

    log = logging.getLogger("libcloudvagrant")

    def __init__(self):
        super(VagrantDriver, self).__init__(key=None)

    def attach_volume(self, node, volume, device=None):
        """Attaches volume to node.

        At the moment only SATA devices of the form ``/dev/sd[a-z]`` are
        accepted as values to parameter ``device``.

        :param node: Node to attach volume to.
        :type node: :class:`VagrantNode`

        :param volume: Volume to attach.
        :type volume: :class:`VagrantVolume`

        :param device: Where the device is exposed, e.g. '/dev/sdb'
        :type device: ``str``

        :rytpe: ``bool``

        """
        if volume.attached_to:
            self.log.warn("Volume %s already attached to %s",
                          volume.name, volume.attached_to)
            return False

        if node.state not in (NodeState.STOPPED, NodeState.UNKNOWN):
            self.log.warn("Cannot attach volumes to running nodes")
            return False

        try:
            with self._catalogue as c:
                node_uuid = c.virtualbox_uuid(node)
                virtualbox.attach_volume(node_uuid, volume.path, device)
                volume.attached_to = node.name
                c.update_volume(volume)
        except:
            self.log.warn("Error attaching %s to %s", volume, node,
                          exc_info=True)
            return False
        self.log.info("Volume '%s' attached to node '%s'",
                      volume.name, node.name)
        return True

    def create_node(self, name, size, image, ex_networks=None, **kwargs):
        """Create a new node instance. This instance will be started
        automatically.

        Requires the following arguments:

        :param name:   String with a name for this new node
        :type name:   ``str``

        :param size:   The size of resources allocated to this node.
        :type size:   :class:`VagrantSize`

        :param image:  OS Image to boot on node.
        :type image:  :class:`VagrantImage`

        Accepts the following non-standard arguments:

        :param ex_networks: The networks to connect this node to.
        :type ex_networks:  ``list`` of :class:`VagrantNetwork`

        All other arguments are ignored.

        """
        if ex_networks is None:
            networks = []
        else:
            networks = ex_networks

        self.log.info("Creating node '%s' ..", name)

        with self._catalogue as c:
            public_ips = [n.allocate_address().to_dict()
                          for n in networks if n.public]
            private_ips = [n.allocate_address().to_dict()
                           for n in networks if not n.public]
            size = size.to_dict()
            image = image.to_dict()
            for n in networks:
                c.update_network(n)
            node = VagrantNode(name=name,
                               public_ips=public_ips,
                               private_ips=private_ips,
                               driver=self,
                               size=size,
                               image=image)
            self.log.debug("create_node(%s): Created object: %s", name, node)
            c.add_node(node)
            c.save()  # Explicit save, so that the next command succeeds
            self.ex_start_node(node)
            self.log.info(".. Node '%s' created", name)
            return node

    def create_volume(self, size, name, **kwargs):
        """Create a new volume.

        :param size: Size of volume in gigabytes (required)
        :type size: ``int``

        :param name: Name of the volume to be created
        :type name: ``str``

        All other arguments are ignored.

        :return: The newly created volume.
        :rtype: :class:`VagrantVolume`

        """
        with self._catalogue as c:
            path = c.volume_path("%s.vdi" % (name,))
            virtualbox.create_volume(path=path, size=size * 1024)
            volume = VagrantVolume(name=name,
                                   size=size,
                                   extra={
                                       "attached_to": None,
                                       "path": path,
                                   },
                                   driver=self)
            c.add_volume(volume)
            self.log.info("Volume '%s' created", name)
            return volume

    def delete_image(self, image):
        """Deletes a node image from a provider.

        :param image: Node image object.
        :type image:  :class:`VagrantImage`

        :return: ``True`` if delete_image was successful, ``False`` otherwise.
        :rtype: ``bool``

        """
        try:
            self._vagrant("box remove --force --provider virtualbox", image.id)
            return True
        except:
            self.log.warn("Cannot remove image %s", image, exc_info=True)
            return False

    def deploy_node(self, **kwargs):
        """Create a new node, and start deployment.

        This method calls ``Node.deploy_node()`` with the default Vagrant SSH
        connection parameters and credentials.

        """
        ssh_config = self._vagrant_ssh_config()
        kwargs["ssh_username"] = ssh_config["user"]
        kwargs["ssh_port"] = ssh_config["port"]
        kwargs["ssh_key"] = ssh_config["key"]
        kwargs["ssh_interface"] = ssh_config["host"]
        self.log.debug("Deploy args: %s", kwargs)
        super(VagrantDriver, self).deploy_node(**kwargs)

    def detach_volume(self, volume):
        """Detaches a volume from a node.

        :param volume: Volume to be detached
        :type volume: :class:`VagrantVolume`

        :rtype: ``bool`

        """
        node = volume.attached_to
        if not node:
            return True
        try:
            with self._catalogue as c:
                node_uuid = c.virtualbox_uuid(node)
                virtualbox.detach_volume(node_uuid, volume.path)
                volume.attached_to = None
                c.update_volume(volume)
                return True
        except:
            self.log.warn("Cannot detach volume %s", volume.name,
                          exc_info=True)
            return False

    def destroy_node(self, node):
        """Destroy a node.

        Volumes attached to this node and networks this node is connected to
        are not destroyed.

        :param node: The node to be destroyed
        :type node: :class:`VagrantNode`

        :return: True if the destroy was successful, False otherwise.
        :rtype: ``bool``

        """
        try:
            with self._catalogue as c:
                self._vagrant("destroy --force", node.name)
                for ip in node._public_ips + node._private_ips:
                    self.log.debug("destroy_node(): Deallocating address %s",
                                   ip)
                    n = c.find_network(ip.network_name)
                    n.deallocate_address(ip.address)
                    c.update_network(n)
                for v in c.get_volumes():
                    if v.attached_to == node.name:
                        self.log.debug("destroy_node(): Detaching %s", v)
                        self.detach_volume(v)
                c.remove_node(node)
            return True
        except:
            self.log.warn("Cannot destroy %s", node.name, exc_info=True)
            return False

    def destroy_volume(self, volume):
        """Destroys a storage volume.

        :param volume: Volume to be destroyed
        :type volume: :class:`VagrantVolume`

        :return: True if the destroy was successful, False otherwise.
        :rtype: ``bool``

        """
        if volume.attached_to:
            self.log.warn("Cannot destroy volume %s: It is attached to %s",
                          volume.name, volume.attached_to)
            return False
        with self._catalogue as c:
            c.remove_volume(volume)
            try:
                os.unlink(volume.path)
            except IOError:
                self.log.warn("Cannot unlink %s", volume.path, exc_info=True)
        return True

    def get_image(self, image_id):
        """Returns a Vagrant image object.

        :param image_id: Image to retrieve (like ``hashicorp/precise64``).
        :type image_id: ``str``

        :return: Image instance on success.
        :rtype :class:`VagrantImage`

        """
        def find_image():
            for image in self.list_images():
                if image.id == image_id:
                    return image

        image = find_image()
        if image:
            return image

        self.log.info("Fetching image '%s' ..", image_id)
        self._vagrant("box add --provider virtualbox", image_id)
        self.log.info(".. Done fetching image '%s'", image_id)

        return find_image()

    def list_images(self, location=None):
        """Lists registered images

        :return: A list of registered images
        :rtype: ``list`` of :class:`VagrantImage`

        """
        images = []
        cur = None
        for line in self._vagrant("box list").strip().split("\n"):
            self.log.debug("Scanning [%s]", line)
            if not line:
                continue
            bits = line.split(",")
            key = bits[2]
            data = bits[3]
            if key == "box-name":
                if cur is not None:
                    images.append(cur)
                cur = {"box-name": data}
            elif key == "box-provider":
                cur["box-provider"] = data
        if cur:
            images.append(cur)

        return [VagrantImage(name=i["box-name"], driver=self)
                for i in images if i["box-provider"] == "virtualbox"]

    def list_nodes(self):
        """Lists all registered nodes.

        :return:  A list of node objects
        :rtype: ``list`` of :class:`VagrantNode`

        """
        with self._catalogue as catalogue:
            nodes = catalogue.get_nodes()
            self.log.debug("Catalogue nodes: %s", nodes)
            return nodes

    def list_sizes(self, location=None):
        """Returns the single size object defined.

        The default size object instructs ``libcloud`` to create a node with
        the same amount of memory and number of CPUs as those of the Vagrant
        image it is created from.

        The ``location`` argument is ignored.

        :return: A list of one single size object
        :rtype: ``list`` of :class:`VagrantSize`

        """
        return [VagrantNodeSize(name="default",
                                ram=0,
                                driver=self,
                                extra={"cpu": 0})]

    def list_volumes(self):
        """Lists all registered storage volumes.

        :rtype: ``list`` of :class:`VagrantVolume`

        """
        with self._catalogue as c:
            ret = c.get_volumes()
            self.log.debug("list_volumes(): Returning %s", ret)
            return ret

    def reboot_node(self, node):
        """Reboot a node.

        :param node: The node to be rebooted
        :type node: :class:`VagrantNode`

        :return: ``True`` if the reboot was successful, otherwise ``False``
        :rtype: ``bool``

        """
        self.log.info("Rebooting node '%s' ..", node.name)
        try:
            self._vagrant("reload --no-provision", node.name)
            self.log.info(".. Node '%s' rebooted", node.name)
            return True
        except:
            self.log.debug("Cannot reload %s", node.name, exc_info=True)

    def wait_until_running(self, nodes, wait_period=3, timeout=600,
                           ssh_interface="public_ips", force_ipv4=True):
        """Block until the provided nodes are considered running.

        Unlike its overridden version, this method does not require any public
        or private IP address to be assigned to the host, since Vagrant's NAT
        interface is always available when a node's state is
        ``NodeState.RUNNING``.

        :param nodes: List of nodes to wait for.
        :type nodes: ``list`` of :class:`VagrantNode`

        :param wait_period: How many seconds to wait between each loop
                            iteration. (default is 3)
        :type wait_period: ``int``

        :param timeout: How many seconds to wait before giving up.
                        (default is 600)
        :type timeout: ``int``

        :param ssh_interface: Ignored parameter.
        :type ssh_interface: ``str``

        :param force_ipv4: Ignored parameter
        :type force_ipv4: ``bool``

        :return: ``[(VagrantNode, ip_addresses)]`` list of tuple of
{'fraction_non_alphanumeric': 0.09599181306498379, 'fraction_numerical': 0.0017056114617090227, 'mean_word_length': 1.8431771894093687, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7454799', 'n_tokens_mistral': 8123, 'n_tokens_neox': 7742, 'n_words': 2554}