<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import utils
from devstack import exceptions
from devstack import shell as sh
from devstack.components import db

LOG = logging.getLogger('devstack.components.nova')

#config files adjusted
API_CONF = 'nova.conf'
PASTE_CONF = 'nova-api-paste.ini'
CONFIGS = [API_CONF, PASTE_CONF]

#this db will be dropped then created
DB_NAME = 'nova'

#id
TYPE = settings.NOVA

#what to start
APP_OPTIONS = {
    settings.NAPI: ['--flagfile', '%CFGFILE%'],
    settings.NCPU: ['--flagfile', '%CFGFILE%'],
    settings.NVOL: ['--flagfile', '%CFGFILE%'],
    'nova-network': ['--flagfile', '%CFGFILE%'],
    'nova-scheduler': ['--flagfile', '%CFGFILE%']
}

#post install cmds that will happen after install
POST_INSTALL_CMDS = [
    {'cmd': ['%BINDIR%/nova-manage', '--flagfile', '%CFGFILE%',
             'db', 'sync']},
    {'cmd': ['%BINDIR%/nova-manage', '--flagfile', '%CFGFILE%',
              'floating', 'create', '%FLOATING_RANGE%']},
    {'cmd': ['%BINDIR%/nova-manage', '--flagfile', '%CFGFILE%',
              'floating', 'create', '--ip_range=%TEST_FLOATING_RANGE%',
              '--pool=%TEST_FLOATING_POOL%']}
]

VG_CHECK_CMD = [
    {'cmd': ['vgs', '%VOLUME_GROUP%'],
     'run_as_root': True}
]

VG_DEV_CMD = [
    {'cmd': ['losetup', '-f', '--show', '%VOLUME_BACKING_FILE%'],
     'run_as_root': True}
]

VG_CREATE_CMD = [
    {'cmd': ['vgcreate', '%VOLUME_GROUP%', '%DEV%'],
     'run_as_root': True}
]

RESTART_TGT_CMD = [
    {'cmd': ['stop', 'tgt'], 'run_as_root': True},
    {'cmd': ['start', 'tgt'], 'run_as_root': True}
]

# In case we need to map names to the image to run
# This map also controls which subcomponent's packages may need to add
APP_NAME_MAP = {
    settings.NAPI: 'nova-api',
    settings.NCPU: 'nova-compute',
    settings.NVOL: 'nova-volume',
}

#subdirs of the checkout/download
BIN_DIR = 'bin'

#These are used by NovaConf
QUANTUM_MANAGER = 'nova.network.quantum.manager.QuantumManager'
NET_MANAGER_TEMPLATE = 'nova.network.manager.%s'
DEF_IMAGE_SERVICE = 'nova.image.glance.GlanceImageService'
DEF_SCHEDULER = 'nova.scheduler.simple.SimpleScheduler'
DEF_GLANCE_PORT = 9292

QUANTUM_OPENSWITCH_OPS = [
    {
      'libvirt_vif_type': ['ethernet'],
      'libvirt_vif_driver': ['nova.virt.libvirt.vif.LibvirtOpenVswitchDriver'],
      'linuxnet_interface_driver': ['nova.network.linux_net.LinuxOVSInterfaceDriver'],
      'quantum_use_dhcp': [],
    }
]


class NovaUninstaller(comp.PythonUninstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)


class NovaInstaller(comp.PythonInstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonInstallComponent.__init__(self, TYPE, *args, **kargs)
        self.git_repo = self.cfg.get("git", "nova_repo")
        self.git_branch = self.cfg.get("git", "nova_branch")
        self.bindir = sh.joinpths(self.appdir, BIN_DIR)
        self.paste_conf_fn = self._get_target_config_name(PASTE_CONF)

    def _get_pkglist(self):
        pkgs = comp.PkgInstallComponent._get_pkglist(self)
        # Walk through the subcomponents (like 'vol' and 'cpu') and add those
        # those packages as well. Let utils.get_pkglist handle any missing
        # entries
        LOG.debug("get_pkglist looking for extras: %s" % (self.component_opts))
        if self.component_opts:
            sub_components = self.component_opts
        else:
            # No subcomponents where explicitly specified, so get all
            sub_components = APP_NAME_MAP.keys()
        # Add the extra dependencies
        for cname in sub_components:
            pkgs.update(utils.get_pkg_list(self.distro, cname))
        return pkgs

    def _get_download_locations(self):
        places = comp.PythonInstallComponent._get_download_locations(self)
        places.append({
            'uri': self.git_repo,
            'branch': self.git_branch,
        })
        return places

    def _get_config_files(self):
        return list(CONFIGS)

    def post_install(self):
        parent_result = comp.PkgInstallComponent.post_install(self)
        LOG.debug("Parent post_install results:%s" % (parent_result))
        #extra actions to do nova setup
        self._setup_db()
        # Need to do db sync and other post install commands
        # set up replacement map for CFGFILE, BINDIR, FLOATING_RANGE,
        # TEST_FLOATING_RANGE, TEST_FLOATING_POOL
        mp = dict()
        mp['BINDIR'] = self.bindir
        mp['CFGFILE'] = sh.joinpths(self.cfgdir, API_CONF)
        mp['FLOATING_RANGE'] = self.cfg.get('nova', 'floating_range')
        mp['TEST_FLOATING_RANGE'] = self.cfg.get('nova', 'test_floating_range')
        mp['TEST_FLOATING_POOL'] = self.cfg.get('nova', 'test_floating_pool')
        utils.execute_template(*POST_INSTALL_CMDS, params=mp, tracewriter=self.tracewriter)
        # check if we need to do the vol subcomponent
        if not self.component_opts or settings.NVOL in self.component_opts:
            # yes, either no subcomponents were specifically requested or it's
            # in the set that was requested
            self._setup_vol_groups()
        return parent_result

    def _setup_db(self):
        LOG.debug("setting up nova DB")
        db.drop_db(self.cfg, DB_NAME)
        db.create_db(self.cfg, DB_NAME)

    def _setup_vol_groups(self):
        LOG.debug("Attempt to setup vol groups")
        mp = dict()
        backing_file = self.cfg.get('nova', 'volume_backing_file')
        # check if we need to have a default backing file
        if not backing_file:
            backing_file = sh.joinpths(self.appdir, 'nova-volumes-backing-file')
        backing_file_size = self.cfg.get('nova', 'volume_backing_file_size')
        if backing_file_size[-1].upper() == 'M':
            backing_file_size = int(backing_file_size[:-1]) * 1024 ** 2
        elif backing_file_size[-1].upper() == 'K':
            backing_file_size = int(backing_file_size[:-1]) * 1024
        elif backing_file_size[-1].upper() == 'B':
            backing_file_size = int(backing_file_size[:-1])
        LOG.debug("backing_file_size:%s" % (backing_file_size))

        mp['VOLUME_GROUP'] = self.cfg.get('nova', 'volume_group')
        mp['VOLUME_BACKING_FILE'] = backing_file
        mp['VOLUME_BACKING_FILE_SIZE'] = backing_file_size
        LOG.debug("params for setup vol group: %s" % (mp))
        try:
            utils.execute_template(*VG_CHECK_CMD, params=mp)
            LOG.debug("Vol group exists")
        except exceptions.ProcessExecutionError as err:
            LOG.debug("Caught expected exception:%s" % (err))
            LOG.info("Need to create vol groups")
            sh.touch_file(backing_file, die_if_there=False, file_size=backing_file_size)
            vg_dev_result = utils.execute_template(*VG_DEV_CMD, params=mp)
            LOG.debug("vg dev result:%s" % (vg_dev_result))
            # String the newlines out of the stdout (which is in the first
            # element of the first (and only) tuple in the response
            mp['DEV'] = vg_dev_result[0][0].replace('\n', '')
            utils.execute_template(*VG_CREATE_CMD, params=mp, tracewriter=self.tracewriter)
        # TODO Now need to check the headings, etc...
        # Finish off by restarting tgt
        utils.execute_template(*RESTART_TGT_CMD, check_exit_code=False, tracewriter=self.tracewriter)

    def _generate_nova_conf(self):
        LOG.debug("Generating dynamic content for nova configuration")
        dirs = dict()
        dirs['app'] = self.appdir
        dirs['cfg'] = self.cfgdir
        dirs['bin'] = self.bindir
        conf_gen = NovaConfigurator(self)
        nova_conf = conf_gen.configure(dirs)
        tgtfn = self._get_target_config_name(API_CONF)
        LOG.info("Writing conf to %s" % (tgtfn))
        LOG.info(nova_conf)
        sh.write_file(tgtfn, nova_conf)
        self.tracewriter.cfg_write(tgtfn)

    def _generate_paste_api_conf(self):
        LOG.info("Setting up %s" % (PASTE_CONF))
        mp = dict()
        mp['SERVICE_TOKEN'] = self.cfg.get("passwords", "service_token")
        (src_fn, contents) = self._get_source_config(PASTE_CONF)
        LOG.info("Replacing parameters in file %s" % (src_fn))
        LOG.debug("Replacements = %s" % (mp))
        contents = utils.param_replace(contents, mp, True)
        LOG.debug("Writing out to %s" % (self.paste_conf_fn))
        sh.write_file(self.paste_conf_fn, contents)
        self.tracewriter.cfg_write(self.paste_conf_fn)

    def _configure_files(self):
        self._generate_nova_conf()
        self._generate_paste_api_conf()
        return len(CONFIGS)


class NovaRuntime(comp.PythonRuntime):
    def __init__(self, *args, **kargs):
        comp.PythonRuntime.__init__(self, TYPE, *args, **kargs)
        self.run_tokens = dict()
        self.run_tokens['CFGFILE'] = sh.joinpths(self.cfgdir, API_CONF)
        LOG.debug("Setting CFGFILE run_token to:%s" % (self.run_tokens['CFGFILE']))

    def _get_apps_to_start(self):
        # Check if component_opts was set to a subset of apps to be started
        LOG.debug("getting list of apps to start")
        apps = list()
        if not self.component_opts and len(self.component_opts) > 0:
            LOG.debug("Attempt to use subset of components:%s" % (self.component_opts))
            # check if the specified sub components exist
            delta = set(self.component_opts) - set(APP_OPTIONS.keys())
            if delta:
                # FIXME, error, something was specified that we don't have
                LOG.error("sub items that we don't know about:%s" % delta)
            else:
                apps = self.component_opts
                LOG.debug("Using specified subcomponents:%s" % (apps))
        else:
            apps = APP_OPTIONS.keys()
            LOG.debug("Using the keys from APP_OPTIONS:%s" % (apps))
<commit_msg>Move work with lv cleanup during install<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import utils
from devstack import exceptions
from devstack import shell as sh
from devstack.components import db

LOG = logging.getLogger('devstack.components.nova')

#config files adjusted
API_CONF = 'nova.conf'
PASTE_CONF = 'nova-api-paste.ini'
CONFIGS = [API_CONF, PASTE_CONF]

#this db will be dropped then created
DB_NAME = 'nova'

#id
TYPE = settings.NOVA

#what to start
APP_OPTIONS = {
    settings.NAPI: ['--flagfile', '%CFGFILE%'],
    settings.NCPU: ['--flagfile', '%CFGFILE%'],
    settings.NVOL: ['--flagfile', '%CFGFILE%'],
    'nova-network': ['--flagfile', '%CFGFILE%'],
    'nova-scheduler': ['--flagfile', '%CFGFILE%']
}

#post install cmds that will happen after install
POST_INSTALL_CMDS = [
    {'cmd': ['%BINDIR%/nova-manage', '--flagfile', '%CFGFILE%',
             'db', 'sync']},
    {'cmd': ['%BINDIR%/nova-manage', '--flagfile', '%CFGFILE%',
              'floating', 'create', '%FLOATING_RANGE%']},
    {'cmd': ['%BINDIR%/nova-manage', '--flagfile', '%CFGFILE%',
              'floating', 'create', '--ip_range=%TEST_FLOATING_RANGE%',
              '--pool=%TEST_FLOATING_POOL%']}
]

VG_CHECK_CMD = [
    {'cmd': ['vgs', '%VOLUME_GROUP%'],
     'run_as_root': True}
]

VG_DEV_CMD = [
    {'cmd': ['losetup', '-f', '--show', '%VOLUME_BACKING_FILE%'],
     'run_as_root': True}
]

VG_CREATE_CMD = [
    {'cmd': ['vgcreate', '%VOLUME_GROUP%', '%DEV%'],
     'run_as_root': True}
]

VG_LVS_CMD = [
    {'cmd': ['lvs', '--noheadings', '-o', 'lv_name', '%VOLUME_GROUP%'],
     'run_as_root': True}
]

VG_LVREMOVE_CMD = [
    {'cmd': ['lvremove', '-f', '%VOLUME_GROUP%/%LV%'],
     'run_as_root': True}
]

RESTART_TGT_CMD = [
    {'cmd': ['stop', 'tgt'], 'run_as_root': True},
    {'cmd': ['start', 'tgt'], 'run_as_root': True}
]

# In case we need to map names to the image to run
# This map also controls which subcomponent's packages may need to add
APP_NAME_MAP = {
    settings.NAPI: 'nova-api',
    settings.NCPU: 'nova-compute',
    settings.NVOL: 'nova-volume',
}

#subdirs of the checkout/download
BIN_DIR = 'bin'

#These are used by NovaConf
QUANTUM_MANAGER = 'nova.network.quantum.manager.QuantumManager'
NET_MANAGER_TEMPLATE = 'nova.network.manager.%s'
DEF_IMAGE_SERVICE = 'nova.image.glance.GlanceImageService'
DEF_SCHEDULER = 'nova.scheduler.simple.SimpleScheduler'
DEF_GLANCE_PORT = 9292

QUANTUM_OPENSWITCH_OPS = [
    {
      'libvirt_vif_type': ['ethernet'],
      'libvirt_vif_driver': ['nova.virt.libvirt.vif.LibvirtOpenVswitchDriver'],
      'linuxnet_interface_driver': ['nova.network.linux_net.LinuxOVSInterfaceDriver'],
      'quantum_use_dhcp': [],
    }
]


class NovaUninstaller(comp.PythonUninstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)


class NovaInstaller(comp.PythonInstallComponent):
    def __init__(self, *args, **kargs):
        comp.PythonInstallComponent.__init__(self, TYPE, *args, **kargs)
        self.git_repo = self.cfg.get("git", "nova_repo")
        self.git_branch = self.cfg.get("git", "nova_branch")
        self.bindir = sh.joinpths(self.appdir, BIN_DIR)
        self.paste_conf_fn = self._get_target_config_name(PASTE_CONF)

    def _get_pkglist(self):
        pkgs = comp.PkgInstallComponent._get_pkglist(self)
        # Walk through the subcomponents (like 'vol' and 'cpu') and add those
        # those packages as well. Let utils.get_pkglist handle any missing
        # entries
        LOG.debug("get_pkglist looking for extras: %s" % (self.component_opts))
        if self.component_opts:
            sub_components = self.component_opts
        else:
            # No subcomponents where explicitly specified, so get all
            sub_components = APP_NAME_MAP.keys()
        # Add the extra dependencies
        for cname in sub_components:
            pkgs.update(utils.get_pkg_list(self.distro, cname))
        return pkgs

    def _get_download_locations(self):
        places = comp.PythonInstallComponent._get_download_locations(self)
        places.append({
            'uri': self.git_repo,
            'branch': self.git_branch,
        })
        return places

    def _get_config_files(self):
        return list(CONFIGS)

    def post_install(self):
        parent_result = comp.PkgInstallComponent.post_install(self)
        LOG.debug("Parent post_install results:%s" % (parent_result))
        #extra actions to do nova setup
        self._setup_db()
        # Need to do db sync and other post install commands
        # set up replacement map for CFGFILE, BINDIR, FLOATING_RANGE,
        # TEST_FLOATING_RANGE, TEST_FLOATING_POOL
        mp = dict()
        mp['BINDIR'] = self.bindir
        mp['CFGFILE'] = sh.joinpths(self.cfgdir, API_CONF)
        mp['FLOATING_RANGE'] = self.cfg.get('nova', 'floating_range')
        mp['TEST_FLOATING_RANGE'] = self.cfg.get('nova', 'test_floating_range')
        mp['TEST_FLOATING_POOL'] = self.cfg.get('nova', 'test_floating_pool')
        utils.execute_template(*POST_INSTALL_CMDS, params=mp, tracewriter=self.tracewriter)
        # check if we need to do the vol subcomponent
        if not self.component_opts or settings.NVOL in self.component_opts:
            # yes, either no subcomponents were specifically requested or it's
            # in the set that was requested
            self._setup_vol_groups()
        return parent_result

    def _setup_db(self):
        LOG.debug("setting up nova DB")
        db.drop_db(self.cfg, DB_NAME)
        db.create_db(self.cfg, DB_NAME)

    def _setup_vol_groups(self):
        LOG.debug("Attempt to setup vol groups")
        mp = dict()
        backing_file = self.cfg.get('nova', 'volume_backing_file')
        # check if we need to have a default backing file
        if not backing_file:
            backing_file = sh.joinpths(self.appdir, 'nova-volumes-backing-file')
        backing_file_size = self.cfg.get('nova', 'volume_backing_file_size')
        if backing_file_size[-1].upper() == 'M':
            backing_file_size = int(backing_file_size[:-1]) * 1024 ** 2
        elif backing_file_size[-1].upper() == 'K':
            backing_file_size = int(backing_file_size[:-1]) * 1024
        elif backing_file_size[-1].upper() == 'B':
            backing_file_size = int(backing_file_size[:-1])
        LOG.debug("backing_file_size:%s" % (backing_file_size))

        mp['VOLUME_GROUP'] = self.cfg.get('nova', 'volume_group')
        mp['VOLUME_BACKING_FILE'] = backing_file
        mp['VOLUME_BACKING_FILE_SIZE'] = backing_file_size
        LOG.debug("params for setup vol group: %s" % (mp))
        try:
            utils.execute_template(*VG_CHECK_CMD, params=mp)
            LOG.debug("Vol group exists")
        except exceptions.ProcessExecutionError as err:
            # Check that the error from VG_CHECK is an expected error
            if err.exit_code != 5:
                raise
            LOG.info("Need to create vol groups")
            sh.touch_file(backing_file, die_if_there=False, file_size=backing_file_size)
            vg_dev_result = utils.execute_template(*VG_DEV_CMD, params=mp)
            LOG.debug("vg dev result:%s" % (vg_dev_result))
            # String the newlines out of the stdout (which is in the first
            # element of the first (and only) tuple in the response
            mp['DEV'] = vg_dev_result[0][0].replace('\n', '')
            utils.execute_template(*VG_CREATE_CMD, params=mp, tracewriter=self.tracewriter)
        # Now check the logical volumes
        self._process_lvs(mp)
        # Finish off by restarting tgt, and ignore any errors
        utils.execute_template(*RESTART_TGT_CMD, check_exit_code=False, tracewriter=self.tracewriter)

    def _process_lvs(self, mp):
        lvs_result = utils.execute_template(*VG_LVS_CMD, params=mp, tracewriter=self.tracewriter)
        LOG.debug("lvs result:%s" % (lvs_result))
        vol_name_prefix = self.cfg.get('nova', 'volume_name_prefix')
        LOG.debug("Using volumne name prefix:%s" % (vol_name_prefix))
        for stdout_line in lvs_result[0][0].split('\n'):
            if stdout_line:
                # Ignore blank lines
                LOG.debug("lvs output line:%s" % (stdout_line))
                if stdout_line.startswith(vol_name_prefix):
                    # TODO still need to implement the following:
                    # tid=`egrep "^tid.+$lv" /proc/net/iet/volume | cut -f1 -d' ' | tr ':' '='`
                    # if [[ -n "$tid" ]]; then
                    #   lun=`egrep "lun.+$lv" /proc/net/iet/volume | cut -f1 -d' ' | tr ':' '=' | tr -d '\t'`
                    #   sudo ietadm --op delete --$tid --$lun
                    # fi
                    # sudo lvremove -f $VOLUME_GROUP/$lv
                    raise exceptions.StackException("lvs magic not yet implemented")
                mp['LV'] = stdout_line
                utils.execute_template(*VG_LVREMOVE_CMD, params=mp, tracewriter=self.tracewriter)

    def _generate_nova_conf(self):
        LOG.debug("Generating dynamic content for nova configuration")
        dirs = dict()
        dirs['app'] = self.appdir
        dirs['cfg'] = self.cfgdir
        dirs['bin'] = self.bindir
        conf_gen = NovaConfigurator(self)
        nova_conf = conf_gen.configure(dirs)
        tgtfn = self._get_target_config_name(API_CONF)
        LOG.info("Writing conf to %s" % (tgtfn))
        LOG.info(nova_conf)
        sh.write_file(tgtfn, nova_conf)
        self.tracewriter.cfg_write(tgtfn)

    def _generate_paste_api_conf(self):
        LOG.info("Setting up %s" % (PASTE_CONF))
        mp = dict()
        mp['SERVICE_TOKEN'] = self.cfg.get("passwords", "service_token")
        (src_fn, contents) = self._get_source_config(PASTE_CONF)
        LOG.info("Replacing parameters in file %s" % (src_fn))
        LOG.debug("Replacements = %s" % (mp))
        contents = utils.param_replace(contents, mp, True)
        LOG.debug("Writing out to %s" % (self.paste_conf_fn))
        sh.write_file(self.paste_conf_fn, contents)
        self.tracewriter.cfg_write(self.paste_conf_fn)

    def _configure_files(self):
        self._generate_nova_conf()
        self._generate_paste_api_conf()
        return len(CONFIGS)


class NovaRuntime(comp.PythonRuntime):
    def __init__(self, *args, **kargs):
        comp.PythonRuntime.__init__(self, TYPE, *args, **kargs)
        self.run_tokens = dict()
        self.run_tokens['CFGFILE'] = sh.joinpths(self.cfgdir, API_CONF)
        LOG.debug("Setting CFGFILE run_token to:%s" % (self.run_tokens['CFGFILE']))

    def _get_apps_to_start(self):
        # Check if component_opts was set to a subset of apps to be started
        LOG.debug("getting list of apps to start")
        apps = list()
        if not self.component_opts and len(self.component_opts) > 0:
            LOG.debug("Attempt to use subset of components:%s" % (self.component_opts))
            # check if the specified sub components exist
            delta = set(self.component_opts) - set(APP_OPTIONS.keys())
            if delta:
                LOG.error("sub items that we don't know about:%s" % delta)
                raise exceptions.BadParamException("Unknown subcomponent specified:%s" % delta)
            else:
                apps = self.component_opts
                LOG.debug("Using specified subcomponents:%s" % (apps))
        else:
            apps = APP_OPTIONS.keys()
            LOG.debug("Using the keys from APP_OPTIONS:%s" % (apps))

{'fraction_non_alphanumeric': 0.1236542736346598, 'fraction_numerical': 0.003094625811794447, 'mean_word_length': 2.9240636223704466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597598', 'n_tokens_mistral': 7485, 'n_tokens_neox': 7341, 'n_words': 1715}