<commit_before># Copyright 2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

""" Provide APIs to tenant management """

import auth
import auth_data_const
import convert
import auth_data
import vmdk_utils
import error_code
import log_config
import logging
from error_code import ErrorCode
from error_code import ErrorInfo
import re

# regex for valid tenant name
VALID_TENANT_NAME_REGEXP = "[a-zA-Z0-9_][a-zA-Z0-9_.-]*"
global valid_tenant_name_reg
valid_tenant_name_reg = re.compile("^" + VALID_TENANT_NAME_REGEXP + "$")

def get_auth_mgr_object():
    """ Get a auth_mgr object which needed to connect to auth DB. """
    # auth.get_auth_mgr will not throw an Exception
    # it will return err_msg when it fails
    err_msg, auth_mgr = auth.get_auth_mgr()
    if err_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, err_msg)
        return error_info, None
    return None, auth_mgr

def get_tenant_from_db(name):
    """
        Get a tenant object with given name
        Return value:
        -- error_code: return None on success or error info on failure
        -- tenant: return tenant object on success or None on failure
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    error_msg, tenant = auth_mgr.get_tenant(name)
    if error_msg:
        error_info = error_code.generate_error_info(error_msg)
    return error_info, tenant

def get_tenant_name(tenant_uuid):
    """
        Get tenant name with given tenant_uuid
        Return value:
        -- error_info: return None on success or error info on failure
        -- tenant_name: return tenant name on success or None on failure
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    error_msg, tenant_name = auth_mgr.get_tenant_name(tenant_uuid)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
    return error_info, tenant_name

def check_tenant_exist(name):
    """ Check tenant with @param name exist or not
        Return value:
        -- Return None if tenant with given name does not exist
        -- Return error_info on failure or the tenant with given name exists
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info

    error_msg, exist_tenant = auth_mgr.get_tenant(name)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
        return error_info

    if exist_tenant:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_ALREADY_EXIST, name)
        return error_info

def create_tenant_in_db(name, description, vms, privileges):
    """
        Create a tenant object in DB
        Return value:
        -- error_info: return None on success or error info on failure
        -- tenant: return tenant object on success or None on failure
    """
    error_info = check_tenant_exist(name)
    if error_info:
        return error_info, None

    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    error_msg, tenant = auth_mgr.create_tenant(name=name,
                                                description=description,
                                                vms=vms,
                                                privileges=privileges)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)

    return error_info, tenant

def get_tenant_list_from_db(name=None):
    """
        List all tenants or tenant with the name specified
        Params:
        -- name: if "name" is specified, return a list of one tenant with name specified
        if "name" is not specified, return a list of all tenants
        Return value:
        -- error_info: return None on success or error info on failure
        -- tenant_list: return a list of tenant objects on success or None on failure
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    if not name:
        error_msg, tenant_list = auth_mgr.list_tenants()
        if error_msg:
            error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
    else:
        error_msg, tenant = auth_mgr.get_tenant(name)
        if error_msg:
            error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
        if error_msg or not tenant:
            tenant_list = []
        else:
            tenant_list = [tenant]

    return error_info, tenant_list

def generate_tuple_from_vm_list(vm_list):
    """ Generate a list of (vm_uuid, vm_name) pair """
    if not vm_list:
        return None, [], []
    vms = []
    error_msg = ""
    not_found_vms = []
    for vm_name in vm_list:
        vm_uuid = vmdk_utils.get_vm_uuid_by_name(vm_name)
        if not vm_uuid:
            err = "Cannot find vm_uuid for vm {0} ".format(vm_name)
            if err:
                error_msg = error_msg + err
                not_found_vms.append(vm_name)
        vms.append((vm_uuid, vm_name))

    if error_msg:
        return error_msg, vms, not_found_vms

    return None, vms, not_found_vms

def default_privileges():
    """ Return a privilege object with default value """
    privileges = [{'datastore': '',
                    'allow_create': 0,
                    'max_volume_size': 0,
                    'usage_quota': 0}]
    return privileges

def set_privileges(allow_create, privileges, value):
    """ set or unset allow_create privileges based on input param  """
    logging.debug("set_privileges: allow_create=%s, privileges=%s, value=%d", allow_create,
                  privileges, value)
    privileges[auth_data_const.COL_ALLOW_CREATE] = value
    return privileges

def validate_string_to_bool(allow_create):
    """
    Validating case insensitive true, false strings
    Return boolean value of the arguement if it is valid,
    else return original value. Also return status if arguement is valid or not
    """
    is_valid = True

    # If already bool, return
    if type(allow_create) is bool:
        return allow_create, is_valid

    allow_create = str(allow_create).lower()

    if allow_create == "true":
        return True, is_valid
    elif allow_create == "false":
        return False, is_valid
    else:
        is_valid = False
        return allow_create, is_valid

def generate_privileges(datastore_url, allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB):
    """ Generate privileges based on input params """
    logging.debug("generate_privileges: datastore_url=%s allow_create=%s"
                  "volume_maxsize_in_MB=%s volume_totalsize_in_MB=%s",
                  datastore_url, allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB)
    privileges = default_privileges()[0]
    privileges[auth_data_const.COL_DATASTORE_URL] = datastore_url

    if allow_create is True:
        privileges = set_privileges(allow_create, privileges, 1)

    if volume_maxsize_in_MB:
        privileges[auth_data_const.COL_MAX_VOLUME_SIZE] = volume_maxsize_in_MB

    if volume_totalsize_in_MB:
        privileges[auth_data_const.COL_USAGE_QUOTA] = volume_totalsize_in_MB

    logging.debug("generate_privileges: privileges=%s", privileges)
    return privileges

def modify_privileges(privileges, allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB):
    """ Modify privileges based on input params """
    logging.debug("modify_privileges: allow_create=%s, volume_maxsize_in_MB=%s, volume_totalsize_in_MB=%s",
                  allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB)

    # If None, don't change the privilege
    # If not None, change accordingly
    if allow_create is not None:
        # allow_create has been validated. It is either True or False
        if allow_create is True:
            privileges = set_privileges(allow_create, privileges, 1)
        else:
            privileges = set_privileges(allow_create, privileges, 0)

    if volume_maxsize_in_MB:
        privileges[auth_data_const.COL_MAX_VOLUME_SIZE] = volume_maxsize_in_MB

    if volume_totalsize_in_MB:
        privileges[auth_data_const.COL_USAGE_QUOTA] = volume_totalsize_in_MB

    return privileges

def generate_privileges_dict(privileges):
    # privileges is a list which is read from auth DB
    # it has the following format
    # (tenant_uuid, datastore_url, allow_create, max_volume_size, usage_quota)
    privileges_dict = {}
    privileges_dict[auth_data_const.COL_DATASTORE_URL] = privileges.datastore_url
    privileges_dict[auth_data_const.COL_ALLOW_CREATE] = privileges.allow_create
    privileges_dict[auth_data_const.COL_MAX_VOLUME_SIZE] = privileges.max_volume_size
    privileges_dict[auth_data_const.COL_USAGE_QUOTA] = privileges.usage_quota
    return privileges_dict

def get_default_datastore_url(name):
    """
        Get default_datastore url for given tenant
        Return value:
        --- error_info: return None on success or error info on failure
        --- default_datastore: return name of default_datastore on success or None on failure
    """
    logging.debug("get_default_datastore: for tenant with name=%s", name)
    error_info, tenant = get_tenant_from_db(name)
    if error_info:
        return error_info, None

    if not tenant:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NOT_EXIST, name)
        return error_info, None

    error_info, auth_mgr = get_auth_mgr_object()

    if error_info:
        return error_info, None

    # if default_datastore is not set for this tenant, default_datastore will be None
    error_msg, default_datastore_url = tenant.get_default_datastore(auth_mgr.conn)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
    return error_info, default_datastore_url

def is_tenant_name_valid(name):
    """ Check given tenant name is valid or not """
    if valid_tenant_name_reg.match(name):
        return True
    else:
        return False

def is_vm_duplicate(vm_list):
    """
    Check if vm names in vm_list contain duplicates
    """

    if len(vm_list) != len(set(vm_list)):
        error_info = error_code.generate_error_info(ErrorCode.VM_DUPLICATE, vm_list)
        logging.error(error_info.msg)
        return error_info

    return None

def _tenant_create(name, description="", vm_list=None, privileges=None):
    """ API to create a tenant """
    logging.debug("_tenant_create: name=%s description=%s vm_list=%s privileges=%s",
                  name, description, vm_list, privileges)
    if not is_tenant_name_valid(name):
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NAME_INVALID,
                                                    name, VALID_TENANT_NAME_REGEXP)
        return error_info, None

    # if param "description" is not set by caller, the default value is empty string
    if not description:
        description = ""

    # VM list can be empty during tenant create. Validate only if it exists
    vms_uuid_list = []
    if vm_list:
        error_info = is_vm_duplicate(vm_list)
        if error_info:
            return error_info, None

        error_msg, vms, not_found_vms = generate_tuple_from_vm_list(vm_list)
        if error_msg:
            not_found_vm_list = ",".join(not_found_vms)
            error_info = error_code.generate_error_info(ErrorCode.VM_NOT_FOUND, not_found_vm_list)
            return error_info, None

        error_info = vm_in_any_tenant(vms)
        if error_info:
            return error_info, None

        logging.debug("_tenant_create: vms=%s", vms)
        vms_uuid_list = [(vm_id) for (vm_id, vm_name) in vms]

    error_info, tenant = create_tenant_in_db(
        name=name,
        description=description,
        vms=vms_uuid_list,
        privileges=privileges)
    if error_info:
        return error_info, None

    return None, tenant

def _tenant_update(name, new_name=None, description=None, default_datastore=None):
    """ API to update a tenant """
    error_info, tenant = get_tenant_from_db(name)
    if error_info:
        return error_info

    if not tenant:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NOT_EXIST, name)
        return error_info

    error_info, auth_mgr = get_auth_mgr_object()

    if error_info:
        return error_info

    if new_name:
        # check whether tenant with new_name already exist or not
        error_info = check_tenant_exist(new_name)
        if error_info:<commit_msg>Disallow renaming the default tenant.<commit_after># Copyright 2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

""" Provide APIs to tenant management """

import auth
import auth_data_const
import convert
import auth_data
import vmdk_utils
import error_code
import log_config
import logging
from error_code import ErrorCode
from error_code import ErrorInfo
import re

# regex for valid tenant name
VALID_TENANT_NAME_REGEXP = "[a-zA-Z0-9_][a-zA-Z0-9_.-]*"
VALID_TENANT_NAMES = 'rename of vm-groups other than _DEFAULT'
global valid_tenant_name_reg
valid_tenant_name_reg = re.compile("^" + VALID_TENANT_NAME_REGEXP + "$")

def get_auth_mgr_object():
    """ Get a auth_mgr object which needed to connect to auth DB. """
    # auth.get_auth_mgr will not throw an Exception
    # it will return err_msg when it fails
    err_msg, auth_mgr = auth.get_auth_mgr()
    if err_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, err_msg)
        return error_info, None
    return None, auth_mgr

def get_tenant_from_db(name):
    """
        Get a tenant object with given name
        Return value:
        -- error_code: return None on success or error info on failure
        -- tenant: return tenant object on success or None on failure
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    error_msg, tenant = auth_mgr.get_tenant(name)
    if error_msg:
        error_info = error_code.generate_error_info(error_msg)
    return error_info, tenant

def get_tenant_name(tenant_uuid):
    """
        Get tenant name with given tenant_uuid
        Return value:
        -- error_info: return None on success or error info on failure
        -- tenant_name: return tenant name on success or None on failure
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    error_msg, tenant_name = auth_mgr.get_tenant_name(tenant_uuid)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
    return error_info, tenant_name

def check_tenant_exist(name):
    """ Check tenant with @param name exist or not
        Return value:
        -- Return None if tenant with given name does not exist
        -- Return error_info on failure or the tenant with given name exists
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info

    error_msg, exist_tenant = auth_mgr.get_tenant(name)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
        return error_info

    if exist_tenant:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_ALREADY_EXIST, name)
        return error_info

def create_tenant_in_db(name, description, vms, privileges):
    """
        Create a tenant object in DB
        Return value:
        -- error_info: return None on success or error info on failure
        -- tenant: return tenant object on success or None on failure
    """
    error_info = check_tenant_exist(name)
    if error_info:
        return error_info, None

    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    error_msg, tenant = auth_mgr.create_tenant(name=name,
                                                description=description,
                                                vms=vms,
                                                privileges=privileges)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)

    return error_info, tenant

def get_tenant_list_from_db(name=None):
    """
        List all tenants or tenant with the name specified
        Params:
        -- name: if "name" is specified, return a list of one tenant with name specified
        if "name" is not specified, return a list of all tenants
        Return value:
        -- error_info: return None on success or error info on failure
        -- tenant_list: return a list of tenant objects on success or None on failure
    """
    error_info, auth_mgr = get_auth_mgr_object()
    if error_info:
        return error_info, None

    if not name:
        error_msg, tenant_list = auth_mgr.list_tenants()
        if error_msg:
            error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
    else:
        error_msg, tenant = auth_mgr.get_tenant(name)
        if error_msg:
            error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
        if error_msg or not tenant:
            tenant_list = []
        else:
            tenant_list = [tenant]

    return error_info, tenant_list

def generate_tuple_from_vm_list(vm_list):
    """ Generate a list of (vm_uuid, vm_name) pair """
    if not vm_list:
        return None, [], []
    vms = []
    error_msg = ""
    not_found_vms = []
    for vm_name in vm_list:
        vm_uuid = vmdk_utils.get_vm_uuid_by_name(vm_name)
        if not vm_uuid:
            err = "Cannot find vm_uuid for vm {0} ".format(vm_name)
            if err:
                error_msg = error_msg + err
                not_found_vms.append(vm_name)
        vms.append((vm_uuid, vm_name))

    if error_msg:
        return error_msg, vms, not_found_vms

    return None, vms, not_found_vms

def default_privileges():
    """ Return a privilege object with default value """
    privileges = [{'datastore': '',
                    'allow_create': 0,
                    'max_volume_size': 0,
                    'usage_quota': 0}]
    return privileges

def set_privileges(allow_create, privileges, value):
    """ set or unset allow_create privileges based on input param  """
    logging.debug("set_privileges: allow_create=%s, privileges=%s, value=%d", allow_create,
                  privileges, value)
    privileges[auth_data_const.COL_ALLOW_CREATE] = value
    return privileges

def validate_string_to_bool(allow_create):
    """
    Validating case insensitive true, false strings
    Return boolean value of the arguement if it is valid,
    else return original value. Also return status if arguement is valid or not
    """
    is_valid = True

    # If already bool, return
    if type(allow_create) is bool:
        return allow_create, is_valid

    allow_create = str(allow_create).lower()

    if allow_create == "true":
        return True, is_valid
    elif allow_create == "false":
        return False, is_valid
    else:
        is_valid = False
        return allow_create, is_valid

def generate_privileges(datastore_url, allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB):
    """ Generate privileges based on input params """
    logging.debug("generate_privileges: datastore_url=%s allow_create=%s"
                  "volume_maxsize_in_MB=%s volume_totalsize_in_MB=%s",
                  datastore_url, allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB)
    privileges = default_privileges()[0]
    privileges[auth_data_const.COL_DATASTORE_URL] = datastore_url

    if allow_create is True:
        privileges = set_privileges(allow_create, privileges, 1)

    if volume_maxsize_in_MB:
        privileges[auth_data_const.COL_MAX_VOLUME_SIZE] = volume_maxsize_in_MB

    if volume_totalsize_in_MB:
        privileges[auth_data_const.COL_USAGE_QUOTA] = volume_totalsize_in_MB

    logging.debug("generate_privileges: privileges=%s", privileges)
    return privileges

def modify_privileges(privileges, allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB):
    """ Modify privileges based on input params """
    logging.debug("modify_privileges: allow_create=%s, volume_maxsize_in_MB=%s, volume_totalsize_in_MB=%s",
                  allow_create, volume_maxsize_in_MB, volume_totalsize_in_MB)

    # If None, don't change the privilege
    # If not None, change accordingly
    if allow_create is not None:
        # allow_create has been validated. It is either True or False
        if allow_create is True:
            privileges = set_privileges(allow_create, privileges, 1)
        else:
            privileges = set_privileges(allow_create, privileges, 0)

    if volume_maxsize_in_MB:
        privileges[auth_data_const.COL_MAX_VOLUME_SIZE] = volume_maxsize_in_MB

    if volume_totalsize_in_MB:
        privileges[auth_data_const.COL_USAGE_QUOTA] = volume_totalsize_in_MB

    return privileges

def generate_privileges_dict(privileges):
    # privileges is a list which is read from auth DB
    # it has the following format
    # (tenant_uuid, datastore_url, allow_create, max_volume_size, usage_quota)
    privileges_dict = {}
    privileges_dict[auth_data_const.COL_DATASTORE_URL] = privileges.datastore_url
    privileges_dict[auth_data_const.COL_ALLOW_CREATE] = privileges.allow_create
    privileges_dict[auth_data_const.COL_MAX_VOLUME_SIZE] = privileges.max_volume_size
    privileges_dict[auth_data_const.COL_USAGE_QUOTA] = privileges.usage_quota
    return privileges_dict

def get_default_datastore_url(name):
    """
        Get default_datastore url for given tenant
        Return value:
        --- error_info: return None on success or error info on failure
        --- default_datastore: return name of default_datastore on success or None on failure
    """
    logging.debug("get_default_datastore: for tenant with name=%s", name)
    error_info, tenant = get_tenant_from_db(name)
    if error_info:
        return error_info, None

    if not tenant:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NOT_EXIST, name)
        return error_info, None

    error_info, auth_mgr = get_auth_mgr_object()

    if error_info:
        return error_info, None

    # if default_datastore is not set for this tenant, default_datastore will be None
    error_msg, default_datastore_url = tenant.get_default_datastore(auth_mgr.conn)
    if error_msg:
        error_info = error_code.generate_error_info(ErrorCode.INTERNAL_ERROR, error_msg)
    return error_info, default_datastore_url

def is_tenant_name_valid(name):
    """ Check given tenant name is valid or not """
    if valid_tenant_name_reg.match(name):
        return True
    else:
        return False

def is_vm_duplicate(vm_list):
    """
    Check if vm names in vm_list contain duplicates
    """

    if len(vm_list) != len(set(vm_list)):
        error_info = error_code.generate_error_info(ErrorCode.VM_DUPLICATE, vm_list)
        logging.error(error_info.msg)
        return error_info

    return None

def _tenant_create(name, description="", vm_list=None, privileges=None):
    """ API to create a tenant """
    logging.debug("_tenant_create: name=%s description=%s vm_list=%s privileges=%s",
                  name, description, vm_list, privileges)
    if not is_tenant_name_valid(name):
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NAME_INVALID,
                                                    name, VALID_TENANT_NAME_REGEXP)
        return error_info, None

    # if param "description" is not set by caller, the default value is empty string
    if not description:
        description = ""

    # VM list can be empty during tenant create. Validate only if it exists
    vms_uuid_list = []
    if vm_list:
        error_info = is_vm_duplicate(vm_list)
        if error_info:
            return error_info, None

        error_msg, vms, not_found_vms = generate_tuple_from_vm_list(vm_list)
        if error_msg:
            not_found_vm_list = ",".join(not_found_vms)
            error_info = error_code.generate_error_info(ErrorCode.VM_NOT_FOUND, not_found_vm_list)
            return error_info, None

        error_info = vm_in_any_tenant(vms)
        if error_info:
            return error_info, None

        logging.debug("_tenant_create: vms=%s", vms)
        vms_uuid_list = [(vm_id) for (vm_id, vm_name) in vms]

    error_info, tenant = create_tenant_in_db(
        name=name,
        description=description,
        vms=vms_uuid_list,
        privileges=privileges)
    if error_info:
        return error_info, None

    return None, tenant

def _tenant_update(name, new_name=None, description=None, default_datastore=None):
    """ API to update a tenant """
    if name == auth_data_const.DEFAULT_TENANT:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NAME_INVALID, name, VALID_TENANT_NAMES)
        return error_info

    error_info, tenant = get_tenant_from_db(name)
    if error_info:
        return error_info

    if not tenant:
        error_info = error_code.generate_error_info(ErrorCode.TENANT_NOT_EXIST, name)
        return error_info

    error_info, auth_mgr = get_auth_mgr_object()

    if error_info:
        return error_info

    if new_name:
        # check whether tenant with new_name already exist or not
        error_info = check_tenant_exist(new_name)
        if error_info:
{'fraction_non_alphanumeric': 0.0693828459745922, 'fraction_numerical': 0.0015034202811395926, 'mean_word_length': 3.027702089009991, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6933472', 'n_tokens_mistral': 8138, 'n_tokens_neox': 8120, 'n_words': 2177}