<commit_before># Copyright 2014-2015 Boxkite Inc.

# This file is part of the DataCats package and is released under
# the terms of the GNU Affero General Public License version 3.0.
# See LICENSE.txt or http://www.fsf.org/licensing/licenses/agpl-3.0.html

from os.path import isdir, exists, join
from os import makedirs, remove, environ
import sys
import subprocess
import shutil
import json
import time
import socket
from sha import sha
from struct import unpack
from ConfigParser import (SafeConfigParser, Error as ConfigParserError)

from datacats import task
from datacats.docker import (web_command, run_container, remove_container,
                             inspect_container, is_boot2docker,
                             docker_host, container_logs, APIError)
from datacats.template import ckan_extension_template
from datacats.scripts import (WEB, SHELL, PASTER, PASTER_CD, PURGE,
    RUN_AS_USER, INSTALL_REQS, CLEAN_VIRTUALENV, INSTALL_PACKAGE,
    COMPILE_LESS, DATAPUSHER, INSTALL_POSTGIS, ADJUST_DEVINI,
    UPDATE_ADD_ADMIN)
from datacats.network import wait_for_service_available, ServiceTimeout
from datacats.password import generate_password
from datacats.error import DatacatsError, WebCommandError, PortAllocatedError

WEB_START_TIMEOUT_SECONDS = 30
DB_INIT_RETRY_SECONDS = 30
DB_INIT_RETRY_DELAY = 2
DOCKER_EXE = 'docker'


class Environment(object):
    """
    DataCats environment settings object

    Create with Environment.new(path) or Environment.load(path)
    """
    def __init__(self, name, target, datadir, site_name, ckan_version=None,
                 port=None, deploy_target=None, site_url=None, always_prod=False,
                 extension_dir='ckan', address=None, remote_server_key=None):
        self.name = name
        self.target = target
        self.datadir = datadir
        self.extension_dir = extension_dir
        self.ckan_version = ckan_version
        # This is the site that all commands will operate on.
        self.site_name = site_name
        self.port = int(port if port else self._choose_port())
        self.address = address
        self.deploy_target = deploy_target
        self.remote_server_key = remote_server_key
        self.site_url = site_url
        self.always_prod = always_prod
        self.sites = None

    def _set_site_name(self, site_name):
        self._site_name = site_name
        self.sitedir = join(self.datadir, 'sites', site_name)

    def _get_site_name(self):
        return self._site_name

    site_name = property(fget=_get_site_name, fset=_set_site_name)

    def _load_sites(self):
        """
        Gets the names of all of the sites from the datadir and stores them
        in self.sites. Also returns this list.
        """
        if not self.sites:
            self.sites = task.list_sites(self.datadir)
        return self.sites

    def save_site(self):
        """
        Save environment settings in the directory that need to be saved
        even when creating only a new sub-site env.
        """
        self._load_sites()
        self.sites.append(self.site_name)

        task.save_new_site(self.site_name, self.sitedir, self.target, self.port,
            self.address, self.site_url, self.passwords)

    def save(self):
        """
        Save environment settings into environment directory, overwriting
        any existing configuration and discarding site config
        """
        task.save_new_environment(self.name, self.datadir, self.target,
            self.ckan_version, self.deploy_target, self.always_prod)

    @classmethod
    def new(cls, path, ckan_version, site_name, **kwargs):
        """
        Return a Environment object with settings for a new project.
        No directories or containers are created by this call.

        :params path: location for new project directory, may be relative
        :params ckan_version: release of CKAN to install
        :params site_name: The name of the site to install database and solr \
                            eventually.

        For additional keyword arguments see the __init__ method.

        Raises DatcatsError if directories or project with same
        name already exits.
        """
        name, datadir, srcdir = task.new_environment_check(path, site_name)
        environment = cls(name, srcdir, datadir, site_name, ckan_version, **kwargs)
        environment._generate_passwords()
        return environment

    @classmethod
    def load(cls, environment_name=None, site_name='primary', data_only=False):
        """
        Return an Environment object based on an existing environnment+site.

        :param environment_name: exising environment name, path or None to
            look in current or parent directories for project
        :param data_only: set to True to only load from data dir, not
            the project dir; Used for purging environment data.

        Raises DatacatsError if environment can't be found or if there is an
        error parsing the environment information.
        """
        srcdir, extension_dir, datadir = task.find_environment_dirs(
            environment_name, data_only)

        if datadir and data_only:
            return cls(environment_name, None, datadir, site_name)

        (datadir, name, ckan_version, always_prod, deploy_target,
            remote_server_key) = task.load_environment(srcdir, datadir)

        (port, address, site_url, passwords
            ) = task.load_site(srcdir, datadir, site_name)

        environment = cls(name, srcdir, datadir, site_name, ckan_version=ckan_version,
                          port=port, deploy_target=deploy_target, site_url=site_url,
                          always_prod=always_prod, address=address,
                          extension_dir=extension_dir,
                          remote_server_key=remote_server_key)

        if passwords:
            environment.passwords = passwords
        else:
            environment._generate_passwords()

        environment._load_sites()
        return environment

    def data_exists(self):
        """
        Return True if the datadir for this environment exists
        """
        return isdir(self.datadir)

    def require_valid_site(self):
        if self.site_name not in self.sites:
            raise DatacatsError('Invalid site name: {}. Valid names are: {}'
                                .format(self.site_name,
                                        ', '.join(self.sites)))

    def data_complete(self):
        """
        Return True if all the expected datadir files are present
        """
        return task.data_complete(self.datadir, self.sitedir,
            self._get_container_name)

    def require_data(self):
        """
        raise a DatacatsError if the datadir or volumes are missing or damaged
        """
        files = task.source_missing(self.target)
        if files:
            raise DatacatsError('Missing files in source directory:\n' +
                                '\n'.join(files))
        if not self.data_exists():
            raise DatacatsError('Environment datadir missing. '
                                'Try "datacats init".')
        if not self.data_complete():
            raise DatacatsError('Environment datadir damaged or volumes '
                                'missing. '
                                'To reset and discard all data use '
                                '"datacats reset"')

    def create_directories(self, create_project_dir=True):
        """
        Call once for new projects to create the initial project directories.
        """
        return task.create_directories(self.datadir, self.sitedir,
            self.target if create_project_dir else None)

    def create_bash_profile(self):
        """
        Create a default .bash_profile for the shell user that
        activates the ckan virtualenv
        """
        with open(self.target + '/.bash_profile', 'w') as prof:
            prof.write('source /usr/lib/ckan/bin/activate\n')

    def _preload_image(self):
        # pylint: disable=no-self-use
        """
        Return the preloaded ckan src and venv image name
        """
        # FIXME: when we support more than one preload image
        # get the preload name from self.ckan_version
        return 'datacats/web:preload-2.3'

    def create_virtualenv(self):
        """
        Populate venv from preloaded image
        """
        return task.create_virtualenv(self.target, self.datadir,
            self._preload_image(), self._get_container_name)

    def clean_virtualenv(self):
        """
        Empty our virtualenv so that new (or older) dependencies may be
        installed
        """
        self.user_run_script(
            script=CLEAN_VIRTUALENV,
            args=[],
            rw_venv=True,
            )

    def create_source(self, datapusher=True):
        """
        Populate ckan directory from preloaded image and copy
        who.ini and schema.xml info conf directory
        """
        task.create_source(self.target, self._preload_image(), datapusher)

    def start_supporting_containers(self):
        """
        Start all supporting containers (containers required for CKAN to
        operate) if they aren't already running.
        """
        task.start_supporting_containers(self.sitedir, self.target,
            self.passwords, self._get_container_name)

    def stop_supporting_containers(self):
        """
        Stop and remove supporting containers (containers that are used by CKAN but don't host
        CKAN or CKAN plugins). This method should *only* be called after CKAN has been stopped
        or behaviour is undefined.
        """
        task.stop_supporting_containers(self._get_container_name)

    def fix_storage_permissions(self):
        """
        Set the owner of all apache storage files to www-data container user
        """
        web_command(
            command='/bin/chown -R www-data: /var/www/storage',
            rw={self.sitedir + '/files': '/var/www/storage'})

    def create_ckan_ini(self):
        """
        Use make-config to generate an initial development.ini file
        """
        self.run_command(
            command='/usr/lib/ckan/bin/paster make-config'
            ' ckan /project/development.ini',
            rw_project=True,
            )

    def update_ckan_ini(self, skin=True):
        """
        Use config-tool to update development.ini with our environment settings

        :param skin: use environment template skin plugin True/False
        """
        command = [
            '/usr/lib/ckan/bin/paster', '--plugin=ckan', 'config-tool',
            '/project/development.ini', '-e',
            'sqlalchemy.url = postgresql://<hidden>',
            'ckan.datastore.read_url = postgresql://<hidden>',
            'ckan.datastore.write_url = postgresql://<hidden>',
            'ckan.datapusher.url = http://datapusher:8800',
            'solr_url = http://solr:8080/solr',
            'ckan.storage_path = /var/www/storage',
            'ckan.plugins = datastore resource_proxy text_view ' +
            ('datapusher ' if exists(self.target + '/datapusher') else '')
            + 'recline_grid_view recline_graph_view'
            + (' {0}_theme'.format(self.name) if skin else ''),
            'ckan.site_title = ' + self.name,
            'ckan.site_logo =',
            'ckan.auth.create_user_via_web = false',
            ]
        self.run_command(command=command, rw_project=True)

    def create_install_template_skin(self):
        """
        Create an example ckan extension for this environment and install it
        """
        ckan_extension_template(self.name, self.target)
        self.install_package_develop('ckanext-' + self.name + 'theme')

    def ckan_db_init(self, retry_seconds=DB_INIT_RETRY_SECONDS):
        """
        Run db init to create all ckan tables

        :param retry_seconds: how long to retry waiting for db to start
        """
        # XXX workaround for not knowing how long we need to wait
        # for postgres to be ready. fix this by changing the postgres
        # entrypoint, or possibly running once with command=/bin/true
        started = time.time()
        while True:
            try:
                self.run_command(
                    '/usr/lib/ckan/bin/paster --plugin=ckan db init '
                    '-c /project/development.ini',
                    db_links=True,
                    clean_up=True,
                    )
                break
            except WebCommandError:
                if started + retry_seconds > time.time():
                    raise
            time.sleep(DB_INIT_RETRY_DELAY)

    def install_postgis_sql(self):
        web_command(
            None,  # use entrypoint to override postgres createdb magic
            entrypoint='/scripts/install_postgis.sh',
            image='datacats/postgres',
            ro={INSTALL_POSTGIS: '/scripts/install_postgis.sh'},
            links={self._get_container_name('postgres'): 'db'},
            )

    def _generate_passwords(self):
        """
        Generate new DB passwords and store them in self.passwords
        """
        self.passwords = {
            'POSTGRES_PASSWORD': <PASSWORD>(),
            'CKAN_PASSWORD': <PASSWORD>(),
            'DATASTORE_RO_PASSWORD': generate_password(),
            'DATASTORE_RW_PASSWORD': <PASSWORD>(),
            'BEAKER_SESSION_SECRET': generate_password(),
            }

    def needs_datapusher(self):
        cp = SafeConfigParser()
        try:
            cp.read(self.target + '/development.ini')
            return 'datapusher' in cp.get('app:main', 'ckan.plugins')
        except ConfigParserError as e:
            raise DatacatsError('Failed to read and parse development.ini: ' + str(e))

    def start_ckan(self, production=False, address='127.0.0.1', log_syslog=False,
                   paster_reload=True):
        """
        Start the apache server or paster serve

        :param production: True for apache, False for paster serve + debug on
        :param address: On Linux, the address to serve from (can be 0.0.0.0 for
                        listening on all addresses)
        :param paster_reload: Instruct paster to watch for file changes
        """
        self.stop_ckan()

        port = self.port

        production = production or self.always_prod
        override_site_url = self.address == '127.0.0.1' and not is_boot2docker()
        command = ['/scripts/web.sh', str(production), str(override_site_url), str(paster_reload)]

        if address != '127.0.0.1' and is_boot2docker():
            raise DatacatsError('Cannot specify address on boot2docker.')

        datapusher = self.needs_datapusher()

        # XXX nasty hack, remove this once we have a lessc command
        # for users (not just for building our preload image)
        if not production:<commit_msg>Add more checks for datapusher.<commit_after># Copyright 2014-2015 Boxkite Inc.

# This file is part of the DataCats package and is released under
# the terms of the GNU Affero General Public License version 3.0.
# See LICENSE.txt or http://www.fsf.org/licensing/licenses/agpl-3.0.html

from os.path import isdir, exists, join
from os import makedirs, remove, environ
import sys
import subprocess
import shutil
import json
import time
import socket
from sha import sha
from struct import unpack
from ConfigParser import (SafeConfigParser, Error as ConfigParserError)

from datacats import task
from datacats.docker import (web_command, run_container, remove_container,
                             inspect_container, is_boot2docker,
                             docker_host, container_logs, APIError)
from datacats.template import ckan_extension_template
from datacats.scripts import (WEB, SHELL, PASTER, PASTER_CD, PURGE,
    RUN_AS_USER, INSTALL_REQS, CLEAN_VIRTUALENV, INSTALL_PACKAGE,
    COMPILE_LESS, DATAPUSHER, INSTALL_POSTGIS, ADJUST_DEVINI,
    UPDATE_ADD_ADMIN)
from datacats.network import wait_for_service_available, ServiceTimeout
from datacats.password import generate_password
from datacats.error import DatacatsError, WebCommandError, PortAllocatedError

WEB_START_TIMEOUT_SECONDS = 30
DB_INIT_RETRY_SECONDS = 30
DB_INIT_RETRY_DELAY = 2
DOCKER_EXE = 'docker'


class Environment(object):
    """
    DataCats environment settings object

    Create with Environment.new(path) or Environment.load(path)
    """
    def __init__(self, name, target, datadir, site_name, ckan_version=None,
                 port=None, deploy_target=None, site_url=None, always_prod=False,
                 extension_dir='ckan', address=None, remote_server_key=None):
        self.name = name
        self.target = target
        self.datadir = datadir
        self.extension_dir = extension_dir
        self.ckan_version = ckan_version
        # This is the site that all commands will operate on.
        self.site_name = site_name
        self.port = int(port if port else self._choose_port())
        self.address = address
        self.deploy_target = deploy_target
        self.remote_server_key = remote_server_key
        self.site_url = site_url
        self.always_prod = always_prod
        self.sites = None

    def _set_site_name(self, site_name):
        self._site_name = site_name
        self.sitedir = join(self.datadir, 'sites', site_name)

    def _get_site_name(self):
        return self._site_name

    site_name = property(fget=_get_site_name, fset=_set_site_name)

    def _load_sites(self):
        """
        Gets the names of all of the sites from the datadir and stores them
        in self.sites. Also returns this list.
        """
        if not self.sites:
            self.sites = task.list_sites(self.datadir)
        return self.sites

    def save_site(self):
        """
        Save environment settings in the directory that need to be saved
        even when creating only a new sub-site env.
        """
        self._load_sites()
        self.sites.append(self.site_name)

        task.save_new_site(self.site_name, self.sitedir, self.target, self.port,
            self.address, self.site_url, self.passwords)

    def save(self):
        """
        Save environment settings into environment directory, overwriting
        any existing configuration and discarding site config
        """
        task.save_new_environment(self.name, self.datadir, self.target,
            self.ckan_version, self.deploy_target, self.always_prod)

    @classmethod
    def new(cls, path, ckan_version, site_name, **kwargs):
        """
        Return a Environment object with settings for a new project.
        No directories or containers are created by this call.

        :params path: location for new project directory, may be relative
        :params ckan_version: release of CKAN to install
        :params site_name: The name of the site to install database and solr \
                            eventually.

        For additional keyword arguments see the __init__ method.

        Raises DatcatsError if directories or project with same
        name already exits.
        """
        name, datadir, srcdir = task.new_environment_check(path, site_name)
        environment = cls(name, srcdir, datadir, site_name, ckan_version, **kwargs)
        environment._generate_passwords()
        return environment

    @classmethod
    def load(cls, environment_name=None, site_name='primary', data_only=False):
        """
        Return an Environment object based on an existing environnment+site.

        :param environment_name: exising environment name, path or None to
            look in current or parent directories for project
        :param data_only: set to True to only load from data dir, not
            the project dir; Used for purging environment data.

        Raises DatacatsError if environment can't be found or if there is an
        error parsing the environment information.
        """
        srcdir, extension_dir, datadir = task.find_environment_dirs(
            environment_name, data_only)

        if datadir and data_only:
            return cls(environment_name, None, datadir, site_name)

        (datadir, name, ckan_version, always_prod, deploy_target,
            remote_server_key) = task.load_environment(srcdir, datadir)

        (port, address, site_url, passwords
            ) = task.load_site(srcdir, datadir, site_name)

        environment = cls(name, srcdir, datadir, site_name, ckan_version=ckan_version,
                          port=port, deploy_target=deploy_target, site_url=site_url,
                          always_prod=always_prod, address=address,
                          extension_dir=extension_dir,
                          remote_server_key=remote_server_key)

        if passwords:
            environment.passwords = passwords
        else:
            environment._generate_passwords()

        environment._load_sites()
        return environment

    def data_exists(self):
        """
        Return True if the datadir for this environment exists
        """
        return isdir(self.datadir)

    def require_valid_site(self):
        if self.site_name not in self.sites:
            raise DatacatsError('Invalid site name: {}. Valid names are: {}'
                                .format(self.site_name,
                                        ', '.join(self.sites)))

    def data_complete(self):
        """
        Return True if all the expected datadir files are present
        """
        return task.data_complete(self.datadir, self.sitedir,
            self._get_container_name)

    def require_data(self):
        """
        raise a DatacatsError if the datadir or volumes are missing or damaged
        """
        files = task.source_missing(self.target)
        if files:
            raise DatacatsError('Missing files in source directory:\n' +
                                '\n'.join(files))
        if not self.data_exists():
            raise DatacatsError('Environment datadir missing. '
                                'Try "datacats init".')
        if not self.data_complete():
            raise DatacatsError('Environment datadir damaged or volumes '
                                'missing. '
                                'To reset and discard all data use '
                                '"datacats reset"')

    def create_directories(self, create_project_dir=True):
        """
        Call once for new projects to create the initial project directories.
        """
        return task.create_directories(self.datadir, self.sitedir,
            self.target if create_project_dir else None)

    def create_bash_profile(self):
        """
        Create a default .bash_profile for the shell user that
        activates the ckan virtualenv
        """
        with open(self.target + '/.bash_profile', 'w') as prof:
            prof.write('source /usr/lib/ckan/bin/activate\n')

    def _preload_image(self):
        # pylint: disable=no-self-use
        """
        Return the preloaded ckan src and venv image name
        """
        # FIXME: when we support more than one preload image
        # get the preload name from self.ckan_version
        return 'datacats/web:preload-2.3'

    def create_virtualenv(self):
        """
        Populate venv from preloaded image
        """
        return task.create_virtualenv(self.target, self.datadir,
            self._preload_image(), self._get_container_name)

    def clean_virtualenv(self):
        """
        Empty our virtualenv so that new (or older) dependencies may be
        installed
        """
        self.user_run_script(
            script=CLEAN_VIRTUALENV,
            args=[],
            rw_venv=True,
            )

    def create_source(self, datapusher=True):
        """
        Populate ckan directory from preloaded image and copy
        who.ini and schema.xml info conf directory
        """
        task.create_source(self.target, self._preload_image(), datapusher)

    def start_supporting_containers(self):
        """
        Start all supporting containers (containers required for CKAN to
        operate) if they aren't already running.
        """
        task.start_supporting_containers(self.sitedir, self.target,
            self.passwords, self._get_container_name)

    def stop_supporting_containers(self):
        """
        Stop and remove supporting containers (containers that are used by CKAN but don't host
        CKAN or CKAN plugins). This method should *only* be called after CKAN has been stopped
        or behaviour is undefined.
        """
        task.stop_supporting_containers(self._get_container_name)

    def fix_storage_permissions(self):
        """
        Set the owner of all apache storage files to www-data container user
        """
        web_command(
            command='/bin/chown -R www-data: /var/www/storage',
            rw={self.sitedir + '/files': '/var/www/storage'})

    def create_ckan_ini(self):
        """
        Use make-config to generate an initial development.ini file
        """
        self.run_command(
            command='/usr/lib/ckan/bin/paster make-config'
            ' ckan /project/development.ini',
            rw_project=True,
            )

    def update_ckan_ini(self, skin=True):
        """
        Use config-tool to update development.ini with our environment settings

        :param skin: use environment template skin plugin True/False
        """
        command = [
            '/usr/lib/ckan/bin/paster', '--plugin=ckan', 'config-tool',
            '/project/development.ini', '-e',
            'sqlalchemy.url = postgresql://<hidden>',
            'ckan.datastore.read_url = postgresql://<hidden>',
            'ckan.datastore.write_url = postgresql://<hidden>',
            'ckan.datapusher.url = http://datapusher:8800',
            'solr_url = http://solr:8080/solr',
            'ckan.storage_path = /var/www/storage',
            'ckan.plugins = datastore resource_proxy text_view ' +
            ('datapusher ' if exists(self.target + '/datapusher') else '')
            + 'recline_grid_view recline_graph_view'
            + (' {0}_theme'.format(self.name) if skin else ''),
            'ckan.site_title = ' + self.name,
            'ckan.site_logo =',
            'ckan.auth.create_user_via_web = false',
            ]
        self.run_command(command=command, rw_project=True)

    def create_install_template_skin(self):
        """
        Create an example ckan extension for this environment and install it
        """
        ckan_extension_template(self.name, self.target)
        self.install_package_develop('ckanext-' + self.name + 'theme')

    def ckan_db_init(self, retry_seconds=DB_INIT_RETRY_SECONDS):
        """
        Run db init to create all ckan tables

        :param retry_seconds: how long to retry waiting for db to start
        """
        # XXX workaround for not knowing how long we need to wait
        # for postgres to be ready. fix this by changing the postgres
        # entrypoint, or possibly running once with command=/bin/true
        started = time.time()
        while True:
            try:
                self.run_command(
                    '/usr/lib/ckan/bin/paster --plugin=ckan db init '
                    '-c /project/development.ini',
                    db_links=True,
                    clean_up=True,
                    )
                break
            except WebCommandError:
                if started + retry_seconds > time.time():
                    raise
            time.sleep(DB_INIT_RETRY_DELAY)

    def install_postgis_sql(self):
        web_command(
            None,  # use entrypoint to override postgres createdb magic
            entrypoint='/scripts/install_postgis.sh',
            image='datacats/postgres',
            ro={INSTALL_POSTGIS: '/scripts/install_postgis.sh'},
            links={self._get_container_name('postgres'): 'db'},
            )

    def _generate_passwords(self):
        """
        Generate new DB passwords and store them in self.passwords
        """
        self.passwords = {
            'POSTGRES_PASSWORD': <PASSWORD>(),
            'CKAN_PASSWORD': <PASSWORD>(),
            'DATASTORE_RO_PASSWORD': <PASSWORD>(),
            'DATASTORE_RW_PASSWORD': generate_password(),
            'BEAKER_SESSION_SECRET': generate_password(),
            }

    def needs_datapusher(self):
        cp = SafeConfigParser()
        try:
            cp.read(self.target + '/development.ini')
            return ('datapusher' in cp.get('app:main', 'ckan.plugins')
                    and isdir(self.target + '/datapusher'))
        except ConfigParserError as e:
            raise DatacatsError('Failed to read and parse development.ini: ' + str(e))

    def start_ckan(self, production=False, address='127.0.0.1', log_syslog=False,
                   paster_reload=True):
        """
        Start the apache server or paster serve

        :param production: True for apache, False for paster serve + debug on
        :param address: On Linux, the address to serve from (can be 0.0.0.0 for
                        listening on all addresses)
        :param paster_reload: Instruct paster to watch for file changes
        """
        self.stop_ckan()

        port = self.port

        production = production or self.always_prod
        override_site_url = self.address == '127.0.0.1' and not is_boot2docker()
        command = ['/scripts/web.sh', str(production), str(override_site_url), str(paster_reload)]

        if address != '127.0.0.1' and is_boot2docker():
            raise DatacatsError('Cannot specify address on boot2docker.')

        datapusher = self.needs_datapusher()

        # XXX nasty hack, remove this once we have a lessc command
        # for users (not just for building our preload image)
        if not production:
{'fraction_non_alphanumeric': 0.0835203107732494, 'fraction_numerical': 0.003616757643749372, 'mean_word_length': 2.2123493975903616, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 2}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7284135', 'n_tokens_mistral': 8477, 'n_tokens_neox': 8199, 'n_words': 2288}