<commit_before>/*
 *  Licensed to the Apache Software Foundation (ASF) under one
 *  or more contributor license agreements.  See the NOTICE file
 *  distributed with this work for additional information
 *  regarding copyright ownership.  The ASF licenses this file
 *  to you under the Apache License, Version 2.0 (the
 *  "License"); you may not use this file except in compliance
 *  with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 *
 */
package org.apache.directory.server.admin;


import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import org.apache.directory.ldap.client.api.LdapConnection;
import org.apache.directory.server.annotations.CreateLdapServer;
import org.apache.directory.server.annotations.CreateTransport;
import org.apache.directory.server.core.annotations.ApplyLdifs;
import org.apache.directory.server.core.integ.AbstractLdapTestUnit;
import org.apache.directory.server.core.integ.FrameworkRunner;
import org.apache.directory.server.core.integ.IntegrationUtils;
import org.apache.directory.shared.ldap.entry.Entry;
import org.apache.directory.shared.ldap.entry.EntryAttribute;
import org.apache.directory.shared.ldap.ldif.LdifUtils;
import org.apache.directory.shared.ldap.message.AddResponse;
import org.apache.directory.shared.ldap.message.ResultCodeEnum;
import org.apache.directory.shared.ldap.schema.SchemaManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;


/**
 * Test cases for the AdministrativePoint Addition operation
 * 
 * We will create the following data structure :
 * <pre>
 * ou=system
 *  |
 *  +-ou=AAP1
 *  |  |
 *  |  +-ou=IAP-CA1
 *  |  |
 *  |  +-ou=IAP-AC1
 *  |  |
 *  |  +-ou=SAP-CA1
 *  |  |
 *  |  +-ou=SAP-AC1
 *  | 
 *  +-ou=SAP-CA2
 *  |
 *  +-ou=SAP-AC2
 *  |
 *  +-ou=entry
 * </pre>
 * 
 * and check that it's present when the server is stopped and restarted
 *
 * @author <a href="mailto:<EMAIL>">Apache Directory Project</a>
 */
@RunWith(FrameworkRunner.class)
@CreateLdapServer(transports =
    { @CreateTransport(protocol = "LDAP") })
@ApplyLdifs(
    {
        // Entry # 1
        "dn: ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: AAP1",
        "administrativeRole: autonomousArea",
        "",
        // Entry # 2
        "dn: ou=IAP-CA1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: IAP-CA1",
        "administrativeRole: collectiveAttributeInnerArea",
        "",
        // Entry # 3
        "dn: ou=IAP-AC1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: IAP-AC1",
        "administrativeRole: accessControlInnerArea",
        "",
        // Entry # 4
        "dn: ou=SAP-CA1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-CA1",
        "administrativeRole: collectiveAttributeSpecificArea",
        "",
        // Entry # 5
        "dn: ou=SAP-AC1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-AC1",
        "administrativeRole: accessControlSpecificArea",
        "",
        // Entry # 6
        "dn: ou=SAP-CA2,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-CA2",
        "administrativeRole: collectiveAttributeSpecificArea",
        "",
        // Entry # 7
        "dn: ou=SAP-AC2,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-AC2",
        "administrativeRole: accessControlSpecificArea",
        "",
        // Entry # 8
        "dn: ou=entry,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: entry",
        ""
    })
public class AdministrativePointAddIT extends AbstractLdapTestUnit
{
    // The shared LDAP connection
    private static LdapConnection connection;

    // A reference to the schema manager
    private static SchemaManager schemaManager;

    @Before
    public void init() throws Exception
    {
        connection = IntegrationUtils.getAdminConnection( service );
        schemaManager = ldapServer.getDirectoryService().getSchemaManager();
    }


    @After
    public void shutdown() throws Exception
    {
        connection.close();
    }


    private EntryAttribute getAdminRole( String dn ) throws Exception
    {
        Entry lookup = connection.lookup( dn, "administrativeRole" );

        assertNotNull( lookup );

        return lookup.get( "administrativeRole" );
    }


    // -------------------------------------------------------------------
    // Test the Add operation
    // -------------------------------------------------------------------
    /**
     * Test the addition of IAPs
     */
    @Test
    public void testAddIAP() throws Exception
    {
        assertTrue( ldapServer.isStarted() );

        // First check that we can't add an IAP in the DIT if there is no
        // parent AAP or SAP with the same role
        Entry entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=entry,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );
        

        AddResponse response = connection.add( entry );

        // It should fail
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );

        // Add the entry under a SAP with the same role which has no parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-CA2,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under a SAP with a different role which has no parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-AC2,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should fail
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );

        // Add the entry under an AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with the same role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=IAP-CA1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with a different role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=IAP-AC1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );

        // Add the entry under an AAP with the same role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-CA1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with a different role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-AC1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );
    }
}<commit_msg>Added some more test for SAP additions<commit_after>/*
 *  Licensed to the Apache Software Foundation (ASF) under one
 *  or more contributor license agreements.  See the NOTICE file
 *  distributed with this work for additional information
 *  regarding copyright ownership.  The ASF licenses this file
 *  to you under the Apache License, Version 2.0 (the
 *  "License"); you may not use this file except in compliance
 *  with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an
 *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *  KIND, either express or implied.  See the License for the
 *  specific language governing permissions and limitations
 *  under the License.
 *
 */
package org.apache.directory.server.admin;


import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import org.apache.directory.ldap.client.api.LdapConnection;
import org.apache.directory.server.annotations.CreateLdapServer;
import org.apache.directory.server.annotations.CreateTransport;
import org.apache.directory.server.core.annotations.ApplyLdifs;
import org.apache.directory.server.core.integ.AbstractLdapTestUnit;
import org.apache.directory.server.core.integ.FrameworkRunner;
import org.apache.directory.server.core.integ.IntegrationUtils;
import org.apache.directory.shared.ldap.entry.Entry;
import org.apache.directory.shared.ldap.entry.EntryAttribute;
import org.apache.directory.shared.ldap.ldif.LdifUtils;
import org.apache.directory.shared.ldap.message.AddResponse;
import org.apache.directory.shared.ldap.message.ResultCodeEnum;
import org.apache.directory.shared.ldap.schema.SchemaManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;


/**
 * Test cases for the AdministrativePoint Addition operation
 * 
 * We will create the following data structure :
 * <pre>
 * ou=system
 *  |
 *  +-ou=AAP1
 *  |  |
 *  |  +-ou=IAP-CA1
 *  |  |
 *  |  +-ou=IAP-AC1
 *  |  |
 *  |  +-ou=SAP-CA1
 *  |  |
 *  |  +-ou=SAP-AC1
 *  | 
 *  +-ou=SAP-CA2
 *  |
 *  +-ou=SAP-AC2
 *  |
 *  +-ou=entry
 * </pre>
 * 
 * and check that it's present when the server is stopped and restarted
 *
 * @author <a href="mailto:<EMAIL>">Apache Directory Project</a>
 */
@RunWith(FrameworkRunner.class)
@CreateLdapServer(transports =
    { @CreateTransport(protocol = "LDAP") })
@ApplyLdifs(
    {
        // Entry # 1
        "dn: ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: AAP1",
        "administrativeRole: autonomousArea",
        "",
        // Entry # 2
        "dn: ou=IAP-CA1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: IAP-CA1",
        "administrativeRole: collectiveAttributeInnerArea",
        "",
        // Entry # 3
        "dn: ou=IAP-AC1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: IAP-AC1",
        "administrativeRole: accessControlInnerArea",
        "",
        // Entry # 4
        "dn: ou=SAP-CA1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-CA1",
        "administrativeRole: collectiveAttributeSpecificArea",
        "",
        // Entry # 5
        "dn: ou=SAP-AC1,ou=AAP1,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-AC1",
        "administrativeRole: accessControlSpecificArea",
        "",
        // Entry # 6
        "dn: ou=SAP-CA2,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-CA2",
        "administrativeRole: collectiveAttributeSpecificArea",
        "",
        // Entry # 7
        "dn: ou=SAP-AC2,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: SAP-AC2",
        "administrativeRole: accessControlSpecificArea",
        "",
        // Entry # 8
        "dn: ou=entry,ou=system",
        "ObjectClass: top",
        "ObjectClass: organizationalUnit",
        "ou: entry",
        ""
    })
public class AdministrativePointAddIT extends AbstractLdapTestUnit
{
    // The shared LDAP connection
    private static LdapConnection connection;

    // A reference to the schema manager
    private static SchemaManager schemaManager;

    @Before
    public void init() throws Exception
    {
        connection = IntegrationUtils.getAdminConnection( service );
        schemaManager = ldapServer.getDirectoryService().getSchemaManager();
    }


    @After
    public void shutdown() throws Exception
    {
        connection.close();
    }


    private EntryAttribute getAdminRole( String dn ) throws Exception
    {
        Entry lookup = connection.lookup( dn, "administrativeRole" );

        assertNotNull( lookup );

        return lookup.get( "administrativeRole" );
    }


    // -------------------------------------------------------------------
    // Test the Add operation
    // -------------------------------------------------------------------
    /**
     * Test the addition of IAPs
     */
    @Test
    public void testAddIAP() throws Exception
    {
        assertTrue( ldapServer.isStarted() );

        // First check that we can't add an IAP in the DIT if there is no
        // parent AAP or SAP with the same role
        Entry entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=entry,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );
        

        AddResponse response = connection.add( entry );

        // It should fail
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );

        // Add the entry under a SAP with the same role which has no parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-CA2,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under a SAP with a different role which has no parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-AC2,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should fail
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );

        // Add the entry under an AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with the same role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=IAP-CA1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with a different role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=IAP-AC1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );

        // Add the entry under an SAP with the same role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-CA1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an SAP with a different role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=IAP-CANew,ou=SAP-AC1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: IAP-CANew",
            "administrativeRole: collectiveAttributeInnerArea"
            );

        response = connection.add( entry );

        // It should fail
        assertNotNull( response );
        assertEquals( ResultCodeEnum.UNWILLING_TO_PERFORM, response.getLdapResult().getResultCode() );
    }


    /**
     * Test the addition of SAPs
     */
    @Test
    public void testAddSAP() throws Exception
    {
        assertTrue( ldapServer.isStarted() );

        // First check that we can add a SAP in the DIT if there is no
        // parent AAP or SAP
        Entry entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=entry,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        AddResponse response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under a SAP with the same role which has no parent AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=SAP-CA2,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under a SAP with a different role which has no parent AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=SAP-AC2,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with the same role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=IAP-CA1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an IAP with a different role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=IAP-AC1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an SAP with the same role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=SAP-CA1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );

        // Add the entry under an SAP with a different role which has a parent AAP
        entry = LdifUtils.createEntry(
            "ou=SAP-CANew,ou=SAP-AC1,ou=AAP1,ou=system",
            "ObjectClass: top",
            "ObjectClass: organizationalUnit",
            "ou: SAP-CANew",
            "administrativeRole: collectiveAttributeSpecificArea"
            );

        response = connection.add( entry );

        // It should succeed
        assertNotNull( response );
        assertEquals( ResultCodeEnum.SUCCESS, response.getLdapResult().getResultCode() );
    }
}
{'fraction_non_alphanumeric': 0.1150492720710491, 'fraction_numerical': 0.004339186503913378, 'mean_word_length': 2.2945891783567136, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1570532', 'n_tokens_mistral': 6975, 'n_tokens_neox': 6789, 'n_words': 1654}