<commit_before># -*- coding: utf-8 -*-
import logging
import httplib as http
import math
from itertools import islice

from flask import request
from modularodm import Q
from modularodm.exceptions import ModularOdmException, ValidationValueError

from framework import status
from framework.utils import iso8601format
from framework.mongo import StoredObject
from framework.flask import redirect
from framework.auth.decorators import must_be_logged_in, collect_auth
from framework.exceptions import HTTPError, PermissionsError

from website import language

from website.util import paths
from website.util import rubeus
from website.exceptions import NodeStateError
from website.project import new_node, new_private_link
from website.project.decorators import (
    must_be_contributor_or_public_but_not_anonymized,
    must_be_contributor_or_public,
    must_be_valid_project,
    must_have_permission,
    must_not_be_registration,
    http_error_if_disk_saving_mode
)
from website.tokens import process_token_or_pass
from website.util.permissions import ADMIN, READ, WRITE, CREATOR_PERMISSIONS
from website.util.rubeus import collect_addon_js
from website.project.model import has_anonymous_link, get_pointer_parent, NodeUpdateError, validate_title
from website.project.forms import NewNodeForm
from website.project.metadata.utils import serialize_meta_schemas
from website.models import Node, Pointer, WatchConfig, PrivateLink, Comment
from website import settings
from website.views import _render_nodes, find_bookmark_collection, validate_page_num
from website.profile import utils
from website.project.licenses import serialize_node_license_record
from website.util.sanitize import strip_html
from website.util import rapply


r_strip_html = lambda collection: rapply(collection, strip_html)
logger = logging.getLogger(__name__)

@must_be_valid_project
@must_have_permission(WRITE)
@must_not_be_registration
def edit_node(auth, node, **kwargs):
    post_data = request.json
    edited_field = post_data.get('name')
    value = post_data.get('value', '')

    new_val = None
    if edited_field == 'title':
        try:
            node.set_title(value, auth=auth)
        except ValidationValueError as e:
            raise HTTPError(
                http.BAD_REQUEST,
                data=dict(message_long=e.message)
            )
        new_val = node.title
    elif edited_field == 'description':
        node.set_description(value, auth=auth)
        new_val = node.description
    elif edited_field == 'category':
        node.category = new_val = value

    try:
        node.save()
    except ValidationValueError as e:
        raise HTTPError(
            http.BAD_REQUEST,
            data=dict(message_long=e.message)
        )
    return {
        'status': 'success',
        'newValue': new_val  # Used by x-editable  widget to reflect changes made by sanitizer
    }


##############################################################################
# New Project
##############################################################################


@must_be_logged_in
def project_new(**kwargs):
    return {}

@must_be_logged_in
def project_new_post(auth, **kwargs):
    user = auth.user

    data = request.get_json()
    title = strip_html(data.get('title'))
    title = title.strip()
    category = data.get('category', 'project')
    template = data.get('template')
    description = strip_html(data.get('description'))
    new_project = {}

    if template:
        original_node = Node.load(template)
        changes = {
            'title': title,
            'category': category,
            'template_node': original_node,
        }

        if description:
            changes['description'] = description

        project = original_node.use_as_template(
            auth=auth,
            changes={
                template: changes,
            }
        )

    else:
        try:
            project = new_node(category, title, user, description)
        except ValidationValueError as e:
            raise HTTPError(
                http.BAD_REQUEST,
                data=dict(message_long=e.message)
            )
        new_project = _view_project(project, auth)
    return {
        'projectUrl': project.url,
        'newNode': new_project['node'] if new_project else None
    }, http.CREATED


@must_be_logged_in
@must_be_valid_project
def project_new_from_template(auth, node, **kwargs):
    new_node = node.use_as_template(
        auth=auth,
        changes=dict(),
    )
    return {'url': new_node.url}, http.CREATED, None


##############################################################################
# New Node
##############################################################################

@must_be_valid_project
@must_have_permission(WRITE)
@must_not_be_registration
def project_new_node(auth, node, **kwargs):
    form = NewNodeForm(request.form)
    user = auth.user
    if form.validate():
        try:
            new_component = new_node(
                title=strip_html(form.title.data),
                user=user,
                category=form.category.data,
                parent=node,
            )
        except ValidationValueError as e:
            raise HTTPError(
                http.BAD_REQUEST,
                data=dict(message_long=e.message)
            )
        redirect_url = node.url
        message = (
            'Your component was created successfully. You can keep working on the project page below, '
            'or go to the new <u><a href={component_url}>component</a></u>.'
        ).format(component_url=new_component.url)
        if form.inherit_contributors.data and node.has_permission(user, WRITE):
            for contributor in node.contributors:
                perm = CREATOR_PERMISSIONS if contributor is user else node.get_permissions(contributor)
                new_component.add_contributor(contributor, permissions=perm, auth=auth)

            new_component.save()
            redirect_url = new_component.url + 'contributors/'
            message = (
                'Your component was created successfully. You can edit the contributor permissions below, '
                'work on your <u><a href={component_url}>component</a></u> or return to the <u> '
                '<a href="{project_url}">project page</a></u>.'
            ).format(component_url=new_component.url, project_url=node.url)
        status.push_status_message(message, kind='info', trust=True)

        return {
            'status': 'success',
        }, 201, None, redirect_url
    else:
        # TODO: This function doesn't seem to exist anymore?
        status.push_errors_to_status(form.errors)
    raise HTTPError(http.BAD_REQUEST, redirect_url=node.url)


@must_be_logged_in
@must_be_valid_project
def project_before_fork(auth, node, **kwargs):
    user = auth.user

    prompts = node.callback('before_fork', user=user)

    if node.has_pointers_recursive:
        prompts.append(
            language.BEFORE_FORK_HAS_POINTERS.format(
                category=node.project_or_component
            )
        )

    return {'prompts': prompts}


@must_be_logged_in
@must_be_valid_project
def project_before_template(auth, node, **kwargs):
    prompts = []

    for addon in node.get_addons():
        if 'node' in addon.config.configs:
            if addon.to_json(auth.user)['addon_full_name']:
                prompts.append(addon.to_json(auth.user)['addon_full_name'])

    return {'prompts': prompts}


@must_be_logged_in
@must_be_valid_project
@http_error_if_disk_saving_mode
def node_fork_page(auth, node, **kwargs):
    try:
        fork = node.fork_node(auth)
    except PermissionsError:
        raise HTTPError(
            http.FORBIDDEN,
            redirect_url=node.url
        )
    message = '{} has been successfully forked.'.format(
        node.project_or_component.capitalize()
    )
    status.push_status_message(message, kind='success', trust=False)
    return fork.url


@must_be_valid_project
@must_be_contributor_or_public_but_not_anonymized
def node_registrations(auth, node, **kwargs):
    return _view_project(node, auth, primary=True)


@must_be_valid_project
@must_be_contributor_or_public_but_not_anonymized
def node_forks(auth, node, **kwargs):
    return _view_project(node, auth, primary=True)


@must_be_valid_project
@must_be_logged_in
@must_have_permission(READ)
def node_setting(auth, node, **kwargs):

    auth.user.update_affiliated_institutions_by_email_domain()
    auth.user.save()
    ret = _view_project(node, auth, primary=True)

    addons_enabled = []
    addon_enabled_settings = []

    for addon in node.get_addons():
        addons_enabled.append(addon.config.short_name)
        if 'node' in addon.config.configs:
            config = addon.to_json(auth.user)
            # inject the MakoTemplateLookup into the template context
            # TODO inject only short_name and render fully client side
            config['template_lookup'] = addon.config.template_lookup
            config['addon_icon_url'] = addon.config.icon_url
            addon_enabled_settings.append(config)

    addon_enabled_settings = sorted(addon_enabled_settings, key=lambda addon: addon['addon_full_name'].lower())

    ret['addon_categories'] = settings.ADDON_CATEGORIES
    ret['addons_available'] = sorted([
        addon
        for addon in settings.ADDONS_AVAILABLE
        if 'node' in addon.owners
        and addon.short_name not in settings.SYSTEM_ADDED_ADDONS['node'] and addon.short_name != 'wiki'
    ], key=lambda addon: addon.full_name.lower())

    for addon in settings.ADDONS_AVAILABLE:
        if 'node' in addon.owners and addon.short_name not in settings.SYSTEM_ADDED_ADDONS['node'] and addon.short_name == 'wiki':
            ret['wiki'] = addon
            break

    ret['addons_enabled'] = addons_enabled
    ret['addon_enabled_settings'] = addon_enabled_settings
    ret['addon_capabilities'] = settings.ADDON_CAPABILITIES
    ret['addon_js'] = collect_node_config_js(node.get_addons())

    ret['include_wiki_settings'] = node.include_wiki_settings(auth.user)

    ret['comments'] = {
        'level': node.comment_level,
    }

    ret['categories'] = settings.NODE_CATEGORY_MAP
    ret['categories'].update({
        'project': 'Project'
    })

    return ret
def collect_node_config_js(addons):
    """Collect webpack bundles for each of the addons' node-cfg.js modules. Return
    the URLs for each of the JS modules to be included on the node addons config page.

    :param list addons: List of node's addon config records.
    """
    js_modules = []
    for addon in addons:
        js_path = paths.resolve_addon_path(addon.config, 'node-cfg.js')
        if js_path:
            js_modules.append(js_path)
    return js_modules


@must_have_permission(WRITE)
@must_not_be_registration
def node_choose_addons(auth, node, **kwargs):
    node.config_addons(request.json, auth)


@must_be_valid_project
@must_have_permission(READ)
def node_contributors(auth, node, **kwargs):
    ret = _view_project(node, auth, primary=True)
    ret['contributors'] = utils.serialize_contributors(node.contributors, node)
    ret['adminContributors'] = utils.serialize_contributors(node.admin_contributors, node, admin=True)
    return ret


@must_have_permission(ADMIN)
def configure_comments(node, **kwargs):
    comment_level = request.json.get('commentLevel')
    if not comment_level:
        node.comment_level = None
    elif comment_level in ['public', 'private']:
        node.comment_level = comment_level
    else:
        raise HTTPError(http.BAD_REQUEST)
    node.save()


##############################################################################
# View Project
##############################################################################

@must_be_valid_project(retractions_valid=True)
@must_be_contributor_or_public
@process_token_or_pass
def view_project(auth, node, **kwargs):
    primary = '/api/v1' not in request.path
    ret = _view_project(node, auth, primary=primary)

    ret['addon_capabilities'] = settings.ADDON_CAPABILITIES
    # Collect the URIs to the static assets for addons that have widgets
    ret['addon_widget_js'] = list(collect_addon_js(
        node,
        filename='widget-cfg.js',
        config_entry='widget'
    ))
    ret.update(rubeus.collect_addon_assets(node))
    return ret

# Reorder components
@must_be_valid_project
@must_not_be_registration
@must_have_permission(WRITE)
def project_reorder_components(node, **kwargs):
    """Reorders the components in a project's component list.

    :param-json list new_list: List of strings that include node IDs and
        node type delimited by ':'.

    """
    # TODO(sloria): Change new_list parameter to be an array of objects
    # {
    #   'newList': {
    #       {'key': 'abc123', 'type': 'node'}
    #   }
    # }
    new_list = [
        tuple(n.split(':'))
        for n in request.json.get('new_list', [])
    ]
    nodes_new = [
        StoredObject.get_collection(schema).load(key)
        for key, schema in new_list
    ]

    valid_nodes = [
        n for n in node.nodes
        if not n.is_deleted
    ]
    deleted_nodes = [
        n for n in node.nodes
        if n.is_deleted
    ]
    if len(valid_nodes) == len(nodes_new) and set(valid_nodes) == set(nodes_new):
        node.nodes = nodes_new + deleted_nodes
        node.save()
        return {}

    logger.error('Got invalid node list in reorder components')
    raise HTTPError(http.BAD_REQUEST)


##############################################################################


@must_be_valid_project
@must_be_contributor_or_public
def project_statistics(auth, node, **kwargs):
    if not (node.can_edit(auth) or node.is_public):
        raise HTTPError(http.FORBIDDEN)
    ret = _view_project(node, auth, primary=True)
    ret['node']['keenio_read_key'] = node.keenio_read_key
    return ret


@must_be_valid_project
@must_be_contributor_or_public
def project_statistics_redirect(auth, node, **kwargs):
    return redirect(node.web_url_for('project_statistics', _guid=True))

###############################################################################
# Make Private/Public
###############################################################################<commit_msg>read-access contributors can see analytics page<commit_after># -*- coding: utf-8 -*-
import logging
import httplib as http
import math
from itertools import islice

from flask import request
from modularodm import Q
from modularodm.exceptions import ModularOdmException, ValidationValueError

from framework import status
from framework.utils import iso8601format
from framework.mongo import StoredObject
from framework.flask import redirect
from framework.auth.decorators import must_be_logged_in, collect_auth
from framework.exceptions import HTTPError, PermissionsError

from website import language

from website.util import paths
from website.util import rubeus
from website.exceptions import NodeStateError
from website.project import new_node, new_private_link
from website.project.decorators import (
    must_be_contributor_or_public_but_not_anonymized,
    must_be_contributor_or_public,
    must_be_valid_project,
    must_have_permission,
    must_not_be_registration,
    http_error_if_disk_saving_mode
)
from website.tokens import process_token_or_pass
from website.util.permissions import ADMIN, READ, WRITE, CREATOR_PERMISSIONS
from website.util.rubeus import collect_addon_js
from website.project.model import has_anonymous_link, get_pointer_parent, NodeUpdateError, validate_title
from website.project.forms import NewNodeForm
from website.project.metadata.utils import serialize_meta_schemas
from website.models import Node, Pointer, WatchConfig, PrivateLink, Comment
from website import settings
from website.views import _render_nodes, find_bookmark_collection, validate_page_num
from website.profile import utils
from website.project.licenses import serialize_node_license_record
from website.util.sanitize import strip_html
from website.util import rapply


r_strip_html = lambda collection: rapply(collection, strip_html)
logger = logging.getLogger(__name__)

@must_be_valid_project
@must_have_permission(WRITE)
@must_not_be_registration
def edit_node(auth, node, **kwargs):
    post_data = request.json
    edited_field = post_data.get('name')
    value = post_data.get('value', '')

    new_val = None
    if edited_field == 'title':
        try:
            node.set_title(value, auth=auth)
        except ValidationValueError as e:
            raise HTTPError(
                http.BAD_REQUEST,
                data=dict(message_long=e.message)
            )
        new_val = node.title
    elif edited_field == 'description':
        node.set_description(value, auth=auth)
        new_val = node.description
    elif edited_field == 'category':
        node.category = new_val = value

    try:
        node.save()
    except ValidationValueError as e:
        raise HTTPError(
            http.BAD_REQUEST,
            data=dict(message_long=e.message)
        )
    return {
        'status': 'success',
        'newValue': new_val  # Used by x-editable  widget to reflect changes made by sanitizer
    }


##############################################################################
# New Project
##############################################################################


@must_be_logged_in
def project_new(**kwargs):
    return {}

@must_be_logged_in
def project_new_post(auth, **kwargs):
    user = auth.user

    data = request.get_json()
    title = strip_html(data.get('title'))
    title = title.strip()
    category = data.get('category', 'project')
    template = data.get('template')
    description = strip_html(data.get('description'))
    new_project = {}

    if template:
        original_node = Node.load(template)
        changes = {
            'title': title,
            'category': category,
            'template_node': original_node,
        }

        if description:
            changes['description'] = description

        project = original_node.use_as_template(
            auth=auth,
            changes={
                template: changes,
            }
        )

    else:
        try:
            project = new_node(category, title, user, description)
        except ValidationValueError as e:
            raise HTTPError(
                http.BAD_REQUEST,
                data=dict(message_long=e.message)
            )
        new_project = _view_project(project, auth)
    return {
        'projectUrl': project.url,
        'newNode': new_project['node'] if new_project else None
    }, http.CREATED


@must_be_logged_in
@must_be_valid_project
def project_new_from_template(auth, node, **kwargs):
    new_node = node.use_as_template(
        auth=auth,
        changes=dict(),
    )
    return {'url': new_node.url}, http.CREATED, None


##############################################################################
# New Node
##############################################################################

@must_be_valid_project
@must_have_permission(WRITE)
@must_not_be_registration
def project_new_node(auth, node, **kwargs):
    form = NewNodeForm(request.form)
    user = auth.user
    if form.validate():
        try:
            new_component = new_node(
                title=strip_html(form.title.data),
                user=user,
                category=form.category.data,
                parent=node,
            )
        except ValidationValueError as e:
            raise HTTPError(
                http.BAD_REQUEST,
                data=dict(message_long=e.message)
            )
        redirect_url = node.url
        message = (
            'Your component was created successfully. You can keep working on the project page below, '
            'or go to the new <u><a href={component_url}>component</a></u>.'
        ).format(component_url=new_component.url)
        if form.inherit_contributors.data and node.has_permission(user, WRITE):
            for contributor in node.contributors:
                perm = CREATOR_PERMISSIONS if contributor is user else node.get_permissions(contributor)
                new_component.add_contributor(contributor, permissions=perm, auth=auth)

            new_component.save()
            redirect_url = new_component.url + 'contributors/'
            message = (
                'Your component was created successfully. You can edit the contributor permissions below, '
                'work on your <u><a href={component_url}>component</a></u> or return to the <u> '
                '<a href="{project_url}">project page</a></u>.'
            ).format(component_url=new_component.url, project_url=node.url)
        status.push_status_message(message, kind='info', trust=True)

        return {
            'status': 'success',
        }, 201, None, redirect_url
    else:
        # TODO: This function doesn't seem to exist anymore?
        status.push_errors_to_status(form.errors)
    raise HTTPError(http.BAD_REQUEST, redirect_url=node.url)


@must_be_logged_in
@must_be_valid_project
def project_before_fork(auth, node, **kwargs):
    user = auth.user

    prompts = node.callback('before_fork', user=user)

    if node.has_pointers_recursive:
        prompts.append(
            language.BEFORE_FORK_HAS_POINTERS.format(
                category=node.project_or_component
            )
        )

    return {'prompts': prompts}


@must_be_logged_in
@must_be_valid_project
def project_before_template(auth, node, **kwargs):
    prompts = []

    for addon in node.get_addons():
        if 'node' in addon.config.configs:
            if addon.to_json(auth.user)['addon_full_name']:
                prompts.append(addon.to_json(auth.user)['addon_full_name'])

    return {'prompts': prompts}


@must_be_logged_in
@must_be_valid_project
@http_error_if_disk_saving_mode
def node_fork_page(auth, node, **kwargs):
    try:
        fork = node.fork_node(auth)
    except PermissionsError:
        raise HTTPError(
            http.FORBIDDEN,
            redirect_url=node.url
        )
    message = '{} has been successfully forked.'.format(
        node.project_or_component.capitalize()
    )
    status.push_status_message(message, kind='success', trust=False)
    return fork.url


@must_be_valid_project
@must_be_contributor_or_public_but_not_anonymized
def node_registrations(auth, node, **kwargs):
    return _view_project(node, auth, primary=True)


@must_be_valid_project
@must_be_contributor_or_public_but_not_anonymized
def node_forks(auth, node, **kwargs):
    return _view_project(node, auth, primary=True)


@must_be_valid_project
@must_be_logged_in
@must_have_permission(READ)
def node_setting(auth, node, **kwargs):

    auth.user.update_affiliated_institutions_by_email_domain()
    auth.user.save()
    ret = _view_project(node, auth, primary=True)

    addons_enabled = []
    addon_enabled_settings = []

    for addon in node.get_addons():
        addons_enabled.append(addon.config.short_name)
        if 'node' in addon.config.configs:
            config = addon.to_json(auth.user)
            # inject the MakoTemplateLookup into the template context
            # TODO inject only short_name and render fully client side
            config['template_lookup'] = addon.config.template_lookup
            config['addon_icon_url'] = addon.config.icon_url
            addon_enabled_settings.append(config)

    addon_enabled_settings = sorted(addon_enabled_settings, key=lambda addon: addon['addon_full_name'].lower())

    ret['addon_categories'] = settings.ADDON_CATEGORIES
    ret['addons_available'] = sorted([
        addon
        for addon in settings.ADDONS_AVAILABLE
        if 'node' in addon.owners
        and addon.short_name not in settings.SYSTEM_ADDED_ADDONS['node'] and addon.short_name != 'wiki'
    ], key=lambda addon: addon.full_name.lower())

    for addon in settings.ADDONS_AVAILABLE:
        if 'node' in addon.owners and addon.short_name not in settings.SYSTEM_ADDED_ADDONS['node'] and addon.short_name == 'wiki':
            ret['wiki'] = addon
            break

    ret['addons_enabled'] = addons_enabled
    ret['addon_enabled_settings'] = addon_enabled_settings
    ret['addon_capabilities'] = settings.ADDON_CAPABILITIES
    ret['addon_js'] = collect_node_config_js(node.get_addons())

    ret['include_wiki_settings'] = node.include_wiki_settings(auth.user)

    ret['comments'] = {
        'level': node.comment_level,
    }

    ret['categories'] = settings.NODE_CATEGORY_MAP
    ret['categories'].update({
        'project': 'Project'
    })

    return ret
def collect_node_config_js(addons):
    """Collect webpack bundles for each of the addons' node-cfg.js modules. Return
    the URLs for each of the JS modules to be included on the node addons config page.

    :param list addons: List of node's addon config records.
    """
    js_modules = []
    for addon in addons:
        js_path = paths.resolve_addon_path(addon.config, 'node-cfg.js')
        if js_path:
            js_modules.append(js_path)
    return js_modules


@must_have_permission(WRITE)
@must_not_be_registration
def node_choose_addons(auth, node, **kwargs):
    node.config_addons(request.json, auth)


@must_be_valid_project
@must_have_permission(READ)
def node_contributors(auth, node, **kwargs):
    ret = _view_project(node, auth, primary=True)
    ret['contributors'] = utils.serialize_contributors(node.contributors, node)
    ret['adminContributors'] = utils.serialize_contributors(node.admin_contributors, node, admin=True)
    return ret


@must_have_permission(ADMIN)
def configure_comments(node, **kwargs):
    comment_level = request.json.get('commentLevel')
    if not comment_level:
        node.comment_level = None
    elif comment_level in ['public', 'private']:
        node.comment_level = comment_level
    else:
        raise HTTPError(http.BAD_REQUEST)
    node.save()


##############################################################################
# View Project
##############################################################################

@must_be_valid_project(retractions_valid=True)
@must_be_contributor_or_public
@process_token_or_pass
def view_project(auth, node, **kwargs):
    primary = '/api/v1' not in request.path
    ret = _view_project(node, auth, primary=primary)

    ret['addon_capabilities'] = settings.ADDON_CAPABILITIES
    # Collect the URIs to the static assets for addons that have widgets
    ret['addon_widget_js'] = list(collect_addon_js(
        node,
        filename='widget-cfg.js',
        config_entry='widget'
    ))
    ret.update(rubeus.collect_addon_assets(node))
    return ret

# Reorder components
@must_be_valid_project
@must_not_be_registration
@must_have_permission(WRITE)
def project_reorder_components(node, **kwargs):
    """Reorders the components in a project's component list.

    :param-json list new_list: List of strings that include node IDs and
        node type delimited by ':'.

    """
    # TODO(sloria): Change new_list parameter to be an array of objects
    # {
    #   'newList': {
    #       {'key': 'abc123', 'type': 'node'}
    #   }
    # }
    new_list = [
        tuple(n.split(':'))
        for n in request.json.get('new_list', [])
    ]
    nodes_new = [
        StoredObject.get_collection(schema).load(key)
        for key, schema in new_list
    ]

    valid_nodes = [
        n for n in node.nodes
        if not n.is_deleted
    ]
    deleted_nodes = [
        n for n in node.nodes
        if n.is_deleted
    ]
    if len(valid_nodes) == len(nodes_new) and set(valid_nodes) == set(nodes_new):
        node.nodes = nodes_new + deleted_nodes
        node.save()
        return {}

    logger.error('Got invalid node list in reorder components')
    raise HTTPError(http.BAD_REQUEST)


##############################################################################


@must_be_valid_project
@must_be_contributor_or_public
def project_statistics(auth, node, **kwargs):
    ret = _view_project(node, auth, primary=True)
    ret['node']['keenio_read_key'] = node.keenio_read_key
    return ret


@must_be_valid_project
@must_be_contributor_or_public
def project_statistics_redirect(auth, node, **kwargs):
    return redirect(node.web_url_for('project_statistics', _guid=True))

###############################################################################
# Make Private/Public
###############################################################################
{'fraction_non_alphanumeric': 0.1374728279924269, 'fraction_numerical': 0.0008414557183928196, 'mean_word_length': 3.402376910016978, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4528345', 'n_tokens_mistral': 8552, 'n_tokens_neox': 8272, 'n_words': 1826}