<commit_before>from __future__ import with_statement

import os
import sys
import string
import types
import functools
import random
import tempfile
from StringIO import StringIO
import shlex
if sys.platform in ('linux2', 'darwin'):
    import pwd

from twisted.python import log
from twisted.internet import defer, error, protocol
from twisted.internet.interfaces import IStreamServerEndpoint, IReactorTime
from twisted.internet.endpoints import TCP4ClientEndpoint, TCP4ServerEndpoint
from zope.interface import implements

from txtorcon.torcontrolprotocol import parse_keywords, TorProtocolFactory
from txtorcon.util import delete_file_or_tree, find_keywords, find_tor_binary
from txtorcon.log import txtorlog
from txtorcon.interface import ITorControlProtocol


class TorNotFound(RuntimeError):
    """
    Raised by launch_tor() in case the tor binary was unspecified and could
    not be found by consulting the shell.
    """


def DefaultTCP4EndpointGenerator(*args, **kw):
    """
    Default generator used to create server-side TCP4ServerEndpoint
    instances. Sets interface='127.0.0.1' instead of the default ('',
    which means all).

    (We use a generator so we can try random ports in case something
    is listening on one; perhaps it would be more straightfoward to
    simply allow an endpoint to be passed to TCPHiddenServiceEndpoint
    instead and fail right away in case something is listening?)
    """

    kw['interface'] = '127.0.0.1'
    return TCP4ServerEndpoint(*args, **kw)


class TCPHiddenServiceEndpoint(object):
    """
    This represents something listening on an arbitrary local port
    that has a Tor configured with a Hidden Service pointing at
    it. :api:`twisted.internet.endpoints.TCP4ServerEndpoint
    <TCP4ServerEndpoint>` is used under the hood to do the local
    listening.

    :ivar onion_uri: the public key, like `timaq4ygg2iegci7.onion`
        which came from the data_dir's `hostname` file

    :ivar onion_private_key: the contents of `data_dir/private_key`

    :ivar data_dir: the data directory, either passed in or created
        with `tempfile.mkstemp`

    :ivar public_port: the port we are advertising
    """

    implements(IStreamServerEndpoint)

    def __init__(self, reactor, config, public_port, data_dir=None,
                 port_generator=functools.partial(random.randrange, 1024, 65534),
                 endpoint_generator=DefaultTCP4EndpointGenerator):
        """
        :param reactor:
            :api:`twisted.internet.interfaces.IReactorTCP` provider

        :param config:
            :class:`txtorcon.TorConfig` instance (doesn't need to be
            bootstrapped). Note that `save()` will be called on this
            at least once. FIXME should I just accept a
            TorControlProtocol instance instead, and create my own
            TorConfig?

        :param public_port:
            The port number we will advertise in the hidden serivces
            directory.

        :param data_dir:
            The hidden-service data directory; if None, one will be
            created in /tmp. This contains the public + private keys
            for the onion uri. If you didn't specify a directory, it's
            up to you to save the public/private keys later if you
            want to re-launch the same hidden service at a different
            time.

        :param port_generator:
            A callable that generates a new random port to try
            listening on. Defaults to `random.randrange(1024, 65535)`

        :param endpoint_generator:
            A callable that generates a new instance of something that
            implements IServerEndpoint (by default TCP4ServerEndpoint)
        """

        self.public_port = public_port
        self.data_dir = data_dir
        self.onion_uri = None
        self.onion_private_key = None
        if self.data_dir:
            self._update_onion(self.data_dir)

        else:
            self.data_dir = tempfile.mkdtemp(prefix='tortmp')

        # shouldn't need to use these
        self.reactor = reactor
        self.config = config
        self.hiddenservice = None
        self.port_generator = port_generator
        self.endpoint_generator = endpoint_generator

        self.retries = 0

        self.defer = defer.Deferred()

    def _update_onion(self, thedir):
        """
        Used internally to update the `onion_uri` and
        `onion_private_key` members.
        """

        hn = os.path.join(thedir, 'hostname')
        pk = os.path.join(thedir, 'private_key')
        try:
            with open(hn, 'r') as hnfile:
                self.onion_uri = hnfile.read().strip()
        except IOError:
            self.onion_uri = None

        try:
            with open(pk, 'r') as pkfile:
                self.onion_private_key = pkfile.read().strip()
        except IOError:
            self.onion_private_key = None

    def _create_hiddenservice(self, arg):
        """
        Internal callback to create a hidden-service config in the
        running Tor (via the `config` member).
        """

        ## FIXME this should be anything that doesn't currently have a
        ## listener, and we should check that....or keep trying random
        ## ports if the "real" listen fails?
        self.listen_port = 80

        self.hiddenservice = HiddenService(self.config, self.data_dir,
                                           ['%d 127.0.0.1:%d' % (self.public_port,
                                                                 self.listen_port)])
        self.config.HiddenServices.append(self.hiddenservice)
        return arg

    def listen(self, protocolfactory):
        """
        Implement :api:`twisted.internet.interfaces.IStreamServerEndpoint <IStreamServerEndpoint>`.

        Returns a Deferred that delivers an
        :api:`twisted.internet.interfaces.IPort` instance that also
        has at least `onion_uri` and `onion_private_key` members set
        (both strings). Really this is just what
        :api:`twisted.internet.endpoints.TCP4ServerEndpoint
        <TCP4ServerEndpoint>`.listen() returned, with a few members set. At
        this point, Tor will have fully started up and successfully
        accepted the hidden service's config.
        """

        self.protocolfactory = protocolfactory
        if self.hiddenservice is None:
            ## we don't have a hidden service yet, but if the config
            ## isn't bootstrapped, we need to wait for it first
            if self.config.post_bootstrap:
                d = self.config.post_bootstrap.addCallback(self._create_hiddenservice)

            else:
                self._create_hiddenservice(None)
                d = self.config.save()

        else:
            ## we already have a hidden service created, but still
            ## want a Deferred so the _create_listener flow is the
            ## same
            d = defer.succeed(self)

        d.addCallback(self._create_listener).addErrback(self._retry_local_port)
        return d

    def _retry_local_port(self, failure):
        """
        Handles :api:`twisted.internet.error.CannotListenError` by
        trying again on another port. After 10 failures, we give up
        and propogate the error.
        """
        failure.trap(error.CannotListenError)

        self.retries += 1
        if self.retries > 10:
            return failure
        self.listen_port = self.port_generator()
        ## we do want to overwrite the whole list, not append
        self.hiddenservice.ports = ['%d 127.0.0.1:%d' % (self.public_port,
                                                         self.listen_port)]
        d = self.config.save()
        d.addCallback(self._create_listener).addErrback(self._retry_local_port)
        return d

    def check_local_endpoint(self, ep):
        """
        This tries to sanity-check ep to see if it's a local address
        or not and returns True or False.

        Currently, 'local' means either it starts with '127.' or is
        '::1'
        """

        if hasattr(ep, '_interface'):
            is_local = ep._interface.startswith('127.') or ep._interface == '::1'
            if not is_local:
                return False
        return True

    def _create_listener(self, proto):
        """
        Creates the local TCP4ServerEndpoint instance, returning a
        Deferred delivering an :api:`twisted.internet.interfaces.IPort` instance that also has
        :meth:`TCPHiddenServiceEndpoint._add_attributes` called
        against it (adds `onion_uri` and `onion_private_key` members).
        """

        self._update_onion(self.hiddenservice.dir)

        self.tcp_endpoint = self.endpoint_generator(self.reactor, self.listen_port)
        if not self.check_local_endpoint(self.tcp_endpoint):
            raise RuntimeError("Endpoint doesn't appear to be a local interface.")
        d = self.tcp_endpoint.listen(self.protocolfactory)
        d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
        return d

    def _add_attributes(self, port):
        port.onion_uri = self.onion_uri
        port.onion_port = self.public_port
        return port


class TorProcessProtocol(protocol.ProcessProtocol):

    def __init__(self, connection_creator, progress_updates=None, config=None,
                 ireactortime=None, timeout=None):
        """
        This will read the output from a Tor process and attempt a
        connection to its control port when it sees any 'Bootstrapped'
        message on stdout. You probably don't need to use this
        directly except as the return value from the
        :func:`txtorcon.launch_tor` method. tor_protocol contains a
        valid :class:`txtorcon.TorControlProtocol` instance by that
        point.

        connection_creator is a callable that should return a Deferred
        that callbacks with a :class:`txtorcon.TorControlProtocol`;
        see :func:`txtorcon.launch_tor` for the default one which is a
        functools.partial that will call
        ``connect(TorProtocolFactory())`` on an appropriate
        :api:`twisted.internet.endpoints.TCP4ClientEndpoint`

        :param connection_creator: A no-parameter callable which
            returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`

        :param progress_updates: A callback which received progress
            updates with three args: percent, tag, summary

        :param config: a TorConfig object to connect to the
            TorControlProtocl from the launched tor (should it succeed)

        :param ireactortime:
            An object implementing IReactorTime (i.e. a reactor) which
            needs to be supplied if you pass a timeout.

        :param timeout:
            An int representing the timeout in seconds. If we are
            unable to reach 100% by this time we will consider the
            setting up of Tor to have failed. Must supply ireactortime
            if you supply this.

        :ivar tor_protocol: The TorControlProtocol instance connected
            to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid
            when the `connected_cb` callback runs.

        :ivar connected_cb: Triggered when the Tor process we
            represent is fully bootstrapped

        """

        self.config = config
        self.tor_protocol = None
        self.connection_creator = connection_creator
        self.progress_updates = progress_updates

        self.connected_cb = defer.Deferred()

        self.attempted_connect = False
        self.to_delete = []
        self.stderr = []
        self.stdout = []

        self._setup_complete = False
        self._did_timeout = False
        self._timeout_delayed_call = None
        if timeout:
            if not ireactortime:
                raise RuntimeError('Must supply an IReactorTime object when supplying a timeout')
            ireactortime = IReactorTime(ireactortime)
            self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)

    def outReceived(self, data):
        """
        :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>` API
        """

        self.stdout.append(data)

        ## minor hack: we can't try this in connectionMade because
        ## that's when the process first starts up so Tor hasn't
        ## opened any ports properly yet. So, we presume that after
        ## its first output we're good-to-go. If this fails, we'll
        ## reset and try again at the next output (see this class'
        ## tor_connection_failed)

        txtorlog.msg(data)
        if not self.attempted_connect and 'Bootstrap' in data:
            self.attempted_connect = True
            d = self.connection_creator()
            d.addCallback(self.tor_connected)
            d.addErrback(self.tor_connection_failed)

    def timeout_expired(self):
        """
        A timeout was supplied during setup, and the time has run out.
        """

        try:
            self.transport.signalProcess('KILL')
        except error.ProcessExitedAlready:
            self.transport.loseConnection()
        self._did_timeout = True

    def errReceived(self, data):
        """
        :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>` API
        """

        self.stderr.append(data)
        self.transport.loseConnection()
        raise RuntimeError("Received stderr output from slave Tor process: " + data)

    def cleanup(self):
        """
        Clean up my temporary files.
        """

        [delete_file_or_tree(f) for f in self.to_delete]
        self.to_delete = []

    def processEnded(self, status):
        """
        :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>` API
        """

        self.cleanup()

        if isinstance(status.value, error.ProcessDone):
            return

        if status.value.exitCode is None:
            if self._did_timeout:
                err = RuntimeError('\n'.join(self.stdout) + "\n\nTimeout waiting for Tor launch..")
            else:
                err = RuntimeError('\n'.join(self.stdout) + "\n\nTor was killed (%s)." % status.value.signal)
        else:
            err = RuntimeError('\n'.join(self.stdout) + "\n\nTor exited with error-code %d" % status.value.exitCode)
        self.connected_cb.errback(err)

    def progress(self, percent, tag, summary):
        """<commit_msg>Send signal SIGTERM instead of SIGKILL when timing out.<commit_after>from __future__ import with_statement

import os
import sys
import string
import types
import functools
import random
import tempfile
from StringIO import StringIO
import shlex
if sys.platform in ('linux2', 'darwin'):
    import pwd

from twisted.python import log
from twisted.internet import defer, error, protocol
from twisted.internet.interfaces import IStreamServerEndpoint, IReactorTime
from twisted.internet.endpoints import TCP4ClientEndpoint, TCP4ServerEndpoint
from zope.interface import implements

from txtorcon.torcontrolprotocol import parse_keywords, TorProtocolFactory
from txtorcon.util import delete_file_or_tree, find_keywords, find_tor_binary
from txtorcon.log import txtorlog
from txtorcon.interface import ITorControlProtocol


class TorNotFound(RuntimeError):
    """
    Raised by launch_tor() in case the tor binary was unspecified and could
    not be found by consulting the shell.
    """


def DefaultTCP4EndpointGenerator(*args, **kw):
    """
    Default generator used to create server-side TCP4ServerEndpoint
    instances. Sets interface='127.0.0.1' instead of the default ('',
    which means all).

    (We use a generator so we can try random ports in case something
    is listening on one; perhaps it would be more straightfoward to
    simply allow an endpoint to be passed to TCPHiddenServiceEndpoint
    instead and fail right away in case something is listening?)
    """

    kw['interface'] = '127.0.0.1'
    return TCP4ServerEndpoint(*args, **kw)


class TCPHiddenServiceEndpoint(object):
    """
    This represents something listening on an arbitrary local port
    that has a Tor configured with a Hidden Service pointing at
    it. :api:`twisted.internet.endpoints.TCP4ServerEndpoint
    <TCP4ServerEndpoint>` is used under the hood to do the local
    listening.

    :ivar onion_uri: the public key, like `timaq4ygg2iegci7.onion`
        which came from the data_dir's `hostname` file

    :ivar onion_private_key: the contents of `data_dir/private_key`

    :ivar data_dir: the data directory, either passed in or created
        with `tempfile.mkstemp`

    :ivar public_port: the port we are advertising
    """

    implements(IStreamServerEndpoint)

    def __init__(self, reactor, config, public_port, data_dir=None,
                 port_generator=functools.partial(random.randrange, 1024, 65534),
                 endpoint_generator=DefaultTCP4EndpointGenerator):
        """
        :param reactor:
            :api:`twisted.internet.interfaces.IReactorTCP` provider

        :param config:
            :class:`txtorcon.TorConfig` instance (doesn't need to be
            bootstrapped). Note that `save()` will be called on this
            at least once. FIXME should I just accept a
            TorControlProtocol instance instead, and create my own
            TorConfig?

        :param public_port:
            The port number we will advertise in the hidden serivces
            directory.

        :param data_dir:
            The hidden-service data directory; if None, one will be
            created in /tmp. This contains the public + private keys
            for the onion uri. If you didn't specify a directory, it's
            up to you to save the public/private keys later if you
            want to re-launch the same hidden service at a different
            time.

        :param port_generator:
            A callable that generates a new random port to try
            listening on. Defaults to `random.randrange(1024, 65535)`

        :param endpoint_generator:
            A callable that generates a new instance of something that
            implements IServerEndpoint (by default TCP4ServerEndpoint)
        """

        self.public_port = public_port
        self.data_dir = data_dir
        self.onion_uri = None
        self.onion_private_key = None
        if self.data_dir:
            self._update_onion(self.data_dir)

        else:
            self.data_dir = tempfile.mkdtemp(prefix='tortmp')

        # shouldn't need to use these
        self.reactor = reactor
        self.config = config
        self.hiddenservice = None
        self.port_generator = port_generator
        self.endpoint_generator = endpoint_generator

        self.retries = 0

        self.defer = defer.Deferred()

    def _update_onion(self, thedir):
        """
        Used internally to update the `onion_uri` and
        `onion_private_key` members.
        """

        hn = os.path.join(thedir, 'hostname')
        pk = os.path.join(thedir, 'private_key')
        try:
            with open(hn, 'r') as hnfile:
                self.onion_uri = hnfile.read().strip()
        except IOError:
            self.onion_uri = None

        try:
            with open(pk, 'r') as pkfile:
                self.onion_private_key = pkfile.read().strip()
        except IOError:
            self.onion_private_key = None

    def _create_hiddenservice(self, arg):
        """
        Internal callback to create a hidden-service config in the
        running Tor (via the `config` member).
        """

        ## FIXME this should be anything that doesn't currently have a
        ## listener, and we should check that....or keep trying random
        ## ports if the "real" listen fails?
        self.listen_port = 80

        self.hiddenservice = HiddenService(self.config, self.data_dir,
                                           ['%d 127.0.0.1:%d' % (self.public_port,
                                                                 self.listen_port)])
        self.config.HiddenServices.append(self.hiddenservice)
        return arg

    def listen(self, protocolfactory):
        """
        Implement :api:`twisted.internet.interfaces.IStreamServerEndpoint <IStreamServerEndpoint>`.

        Returns a Deferred that delivers an
        :api:`twisted.internet.interfaces.IPort` instance that also
        has at least `onion_uri` and `onion_private_key` members set
        (both strings). Really this is just what
        :api:`twisted.internet.endpoints.TCP4ServerEndpoint
        <TCP4ServerEndpoint>`.listen() returned, with a few members set. At
        this point, Tor will have fully started up and successfully
        accepted the hidden service's config.
        """

        self.protocolfactory = protocolfactory
        if self.hiddenservice is None:
            ## we don't have a hidden service yet, but if the config
            ## isn't bootstrapped, we need to wait for it first
            if self.config.post_bootstrap:
                d = self.config.post_bootstrap.addCallback(self._create_hiddenservice)

            else:
                self._create_hiddenservice(None)
                d = self.config.save()

        else:
            ## we already have a hidden service created, but still
            ## want a Deferred so the _create_listener flow is the
            ## same
            d = defer.succeed(self)

        d.addCallback(self._create_listener).addErrback(self._retry_local_port)
        return d

    def _retry_local_port(self, failure):
        """
        Handles :api:`twisted.internet.error.CannotListenError` by
        trying again on another port. After 10 failures, we give up
        and propogate the error.
        """
        failure.trap(error.CannotListenError)

        self.retries += 1
        if self.retries > 10:
            return failure
        self.listen_port = self.port_generator()
        ## we do want to overwrite the whole list, not append
        self.hiddenservice.ports = ['%d 127.0.0.1:%d' % (self.public_port,
                                                         self.listen_port)]
        d = self.config.save()
        d.addCallback(self._create_listener).addErrback(self._retry_local_port)
        return d

    def check_local_endpoint(self, ep):
        """
        This tries to sanity-check ep to see if it's a local address
        or not and returns True or False.

        Currently, 'local' means either it starts with '127.' or is
        '::1'
        """

        if hasattr(ep, '_interface'):
            is_local = ep._interface.startswith('127.') or ep._interface == '::1'
            if not is_local:
                return False
        return True

    def _create_listener(self, proto):
        """
        Creates the local TCP4ServerEndpoint instance, returning a
        Deferred delivering an :api:`twisted.internet.interfaces.IPort` instance that also has
        :meth:`TCPHiddenServiceEndpoint._add_attributes` called
        against it (adds `onion_uri` and `onion_private_key` members).
        """

        self._update_onion(self.hiddenservice.dir)

        self.tcp_endpoint = self.endpoint_generator(self.reactor, self.listen_port)
        if not self.check_local_endpoint(self.tcp_endpoint):
            raise RuntimeError("Endpoint doesn't appear to be a local interface.")
        d = self.tcp_endpoint.listen(self.protocolfactory)
        d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
        return d

    def _add_attributes(self, port):
        port.onion_uri = self.onion_uri
        port.onion_port = self.public_port
        return port


class TorProcessProtocol(protocol.ProcessProtocol):

    def __init__(self, connection_creator, progress_updates=None, config=None,
                 ireactortime=None, timeout=None):
        """
        This will read the output from a Tor process and attempt a
        connection to its control port when it sees any 'Bootstrapped'
        message on stdout. You probably don't need to use this
        directly except as the return value from the
        :func:`txtorcon.launch_tor` method. tor_protocol contains a
        valid :class:`txtorcon.TorControlProtocol` instance by that
        point.

        connection_creator is a callable that should return a Deferred
        that callbacks with a :class:`txtorcon.TorControlProtocol`;
        see :func:`txtorcon.launch_tor` for the default one which is a
        functools.partial that will call
        ``connect(TorProtocolFactory())`` on an appropriate
        :api:`twisted.internet.endpoints.TCP4ClientEndpoint`

        :param connection_creator: A no-parameter callable which
            returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`

        :param progress_updates: A callback which received progress
            updates with three args: percent, tag, summary

        :param config: a TorConfig object to connect to the
            TorControlProtocl from the launched tor (should it succeed)

        :param ireactortime:
            An object implementing IReactorTime (i.e. a reactor) which
            needs to be supplied if you pass a timeout.

        :param timeout:
            An int representing the timeout in seconds. If we are
            unable to reach 100% by this time we will consider the
            setting up of Tor to have failed. Must supply ireactortime
            if you supply this.

        :ivar tor_protocol: The TorControlProtocol instance connected
            to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid
            when the `connected_cb` callback runs.

        :ivar connected_cb: Triggered when the Tor process we
            represent is fully bootstrapped

        """

        self.config = config
        self.tor_protocol = None
        self.connection_creator = connection_creator
        self.progress_updates = progress_updates

        self.connected_cb = defer.Deferred()

        self.attempted_connect = False
        self.to_delete = []
        self.stderr = []
        self.stdout = []

        self._setup_complete = False
        self._did_timeout = False
        self._timeout_delayed_call = None
        if timeout:
            if not ireactortime:
                raise RuntimeError('Must supply an IReactorTime object when supplying a timeout')
            ireactortime = IReactorTime(ireactortime)
            self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)

    def outReceived(self, data):
        """
        :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>` API
        """

        self.stdout.append(data)

        ## minor hack: we can't try this in connectionMade because
        ## that's when the process first starts up so Tor hasn't
        ## opened any ports properly yet. So, we presume that after
        ## its first output we're good-to-go. If this fails, we'll
        ## reset and try again at the next output (see this class'
        ## tor_connection_failed)

        txtorlog.msg(data)
        if not self.attempted_connect and 'Bootstrap' in data:
            self.attempted_connect = True
            d = self.connection_creator()
            d.addCallback(self.tor_connected)
            d.addErrback(self.tor_connection_failed)

    def timeout_expired(self):
        """
        A timeout was supplied during setup, and the time has run out.
        """

        try:
            self.transport.signalProcess('TERM')
        except error.ProcessExitedAlready:
            self.transport.loseConnection()
        self._did_timeout = True

    def errReceived(self, data):
        """
        :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>` API
        """

        self.stderr.append(data)
        self.transport.loseConnection()
        raise RuntimeError("Received stderr output from slave Tor process: " + data)

    def cleanup(self):
        """
        Clean up my temporary files.
        """

        [delete_file_or_tree(f) for f in self.to_delete]
        self.to_delete = []

    def processEnded(self, status):
        """
        :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>` API
        """

        self.cleanup()

        if isinstance(status.value, error.ProcessDone) and not self._did_timeout:
            return

        if status.value.exitCode is None:
            if self._did_timeout:
                err = RuntimeError('\n'.join(self.stdout) + "\n\nTimeout waiting for Tor launch..")
            else:
                err = RuntimeError('\n'.join(self.stdout) + "\n\nTor was killed (%s)." % status.value.signal)
        else:
            err = RuntimeError('\n'.join(self.stdout) + "\n\nTor exited with error-code %d" % status.value.exitCode)
        self.connected_cb.errback(err)

    def progress(self, percent, tag, summary):
        """
{'fraction_non_alphanumeric': 0.07594501718213058, 'fraction_numerical': 0.0053608247422680414, 'mean_word_length': 2.3388022028453417, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6776597', 'n_tokens_mistral': 7917, 'n_tokens_neox': 7624, 'n_words': 2461}