<commit_before>/*
 * Copyright 2012 Shared Learning Collaborative, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package org.slc.sli.api.resources.security;

import java.io.File;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;

import javax.annotation.PostConstruct;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.UriInfo;

import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import org.slc.sli.api.config.EntityDefinitionStore;
import org.slc.sli.api.constants.ParameterConstants;
import org.slc.sli.api.init.RoleInitializer;
import org.slc.sli.api.representation.EntityBody;
import org.slc.sli.api.resources.Resource;
import org.slc.sli.api.resources.v1.DefaultCrudEndpoint;
import org.slc.sli.api.security.context.resolver.RealmHelper;
import org.slc.sli.api.service.EntityService;
import org.slc.sli.api.util.SecurityUtil;
import org.slc.sli.domain.NeutralCriteria;
import org.slc.sli.domain.NeutralQuery;
import org.slc.sli.domain.enums.Right;

/**
 *
 * Provides CRUD operations on registered application through the /tenants path.
 *
 * @author
 */
@Component
@Scope("request")
@Path("tenants")
@Produces({ Resource.JSON_MEDIA_TYPE + ";charset=utf-8" })
public class TenantResourceImpl extends DefaultCrudEndpoint implements TenantResource {

    @Autowired
    private EntityDefinitionStore store;

    @Autowired
    private RoleInitializer roleInitializer;

    @Value("${sli.tenant.landingZoneMountPoint}")
    private String landingZoneMountPoint;

    @Value("${sli.tenant.ingestionServers}")
    private String ingestionServers;

    @Autowired
    private RealmHelper realmHelper;

    private List<String> ingestionServerList;

    /* this is only available for unit testing */
    public void setIngestionServerList(List<String> testList) {
        ingestionServerList = testList;
    }

    //private Random random = new Random(System.currentTimeMillis());

    @PostConstruct
    protected void init() {
        ingestionServerList = Arrays.asList(ingestionServers.split(","));
    }

    public static final String UUID = "uuid";
    public static final String RESOURCE_NAME = "tenant";
    public static final String TENANT_ID = "tenantId";
    public static final String LZ = "landingZone";
    public static final String LZ_EDUCATION_ORGANIZATION = "educationOrganization";
    public static final String LZ_INGESTION_SERVER = "ingestionServer";
    public static final String LZ_PATH = "path";
    public static final String LZ_USER_NAMES = "userNames";
    public static final String LZ_DESC = "desc";
    public static final String LZ_INGESTION_SERVER_LOCALHOST = "localhost";
    public static final String LZ_PRELOAD              = "preload";
    public static final String LZ_PRELOAD_FILES        = "files";
    public static final String LZ_PRELOAD_STATUS       = "status";
    public static final String LZ_PRELOAD_STATUS_READY = "ready";


    @Autowired
    public TenantResourceImpl(EntityDefinitionStore entityDefs) {
        super(entityDefs, RESOURCE_NAME);
        store = entityDefs;
    }

    @Override
    @POST
    public Response create(EntityBody newTenant, @Context HttpHeaders headers, @Context final UriInfo uriInfo) {

        // Tenants can not be created using this class. They will be created via OnboardingResource
        return SecurityUtil.forbiddenResponse();
    }

    @Override
    public LandingZoneInfo createLandingZone(String tenantId, String edOrgId, List<String> preloadFiles, boolean isSandbox)
            throws TenantResourceCreationException {
        String newTenantId = createLandingZone(tenantId, edOrgId, null, null, preloadFiles, isSandbox);
        EntityService tenantService = store.lookupByResourceName(RESOURCE_NAME).getService();
        EntityBody newTenant = tenantService.get(newTenantId);
        @SuppressWarnings("unchecked")
        List<Map<String, Object>> newLzs = (List<Map<String, Object>>) newTenant.get(LZ);
        for (Map<String, Object> lz : newLzs) {
            if (edOrgId.equals(lz.get(LZ_EDUCATION_ORGANIZATION))) {
                return new LandingZoneInfo((String) lz.get(LZ_PATH), (String) lz.get(LZ_INGESTION_SERVER));
            }
        }
        throw new TenantResourceCreationException(Status.INTERNAL_SERVER_ERROR, "Failed to find landing zone information after creation.");
    }

    @SuppressWarnings({ "unchecked" })
    protected String createLandingZone(EntityBody newTenant, boolean isSandbox)
            throws TenantResourceCreationException {
        List<Map<String, Object>> newLzs = (List<Map<String, Object>>) newTenant.get(LZ);

        // NOTE: OnboardingResource may only send in one at a time
        if (1 != newLzs.size()) {
            throw new TenantResourceCreationException(Status.BAD_REQUEST,
                    "Only one landing zone may be provisioned at a time.  Please submit your requests individually.");
        }

        String tenantId = (String) newTenant.get(TENANT_ID);

        Map<String, Object> newLz = newLzs.get(0);
        String newEdOrg = (String) newLz.get(LZ_EDUCATION_ORGANIZATION);

        Map<String, Object> preload = (Map<String, Object>) newLz.get(LZ_PRELOAD);
        List<String> preloadFiles = (preload == null) ? (null) : (List<String>) preload.get(LZ_PRELOAD_FILES);

        return createLandingZone(tenantId, newEdOrg, (String) newLz.get(LZ_DESC),
                (List<String>) newLz.get(LZ_USER_NAMES), preloadFiles, isSandbox);
    }

    @SuppressWarnings({ "rawtypes", "unchecked" })
    protected String createLandingZone(final String tenantId, String edOrgId, String desc, List<String> userNames, List<String> preloadFiles, boolean isSandbox)
            throws TenantResourceCreationException {

        // get the exisint tenant resource
        EntityService tenantService = store.lookupByResourceName(RESOURCE_NAME).getService();
        NeutralQuery query = new NeutralQuery();
        query.addCriteria(new NeutralCriteria(TENANT_ID, "=", tenantId));

        String ingestionServer = findLeastLoadedIngestionServer();
        File inboundDirFile = new File(landingZoneMountPoint);
        File fullPath = new File(inboundDirFile, tenantId + "/" + DigestUtils.sha256Hex(edOrgId));
        String path = fullPath.getAbsolutePath();

        //resolve localhost ingestion server to the current server name
        if (ingestionServer.equals(LZ_INGESTION_SERVER_LOCALHOST)) {
            try {
                ingestionServer = InetAddress.getLocalHost().getHostName();
            } catch (UnknownHostException e) {
                throw new TenantResourceCreationException(Status.INTERNAL_SERVER_ERROR,
                        "Failed to resolve ingestion server for " + LZ_INGESTION_SERVER_LOCALHOST + ".");
            }
        }

        // look up ids of existing tenant entries
        List<String> existingIds = new ArrayList<String>();
        for (String id : tenantService.listIds(query)) {
            existingIds.add(id);
        }

        // If no tenant already exists, create one
        if (existingIds.size() == 0) {
            EntityBody newTenant = new EntityBody();
            newTenant.put(TENANT_ID, tenantId);
            Map<String, Object> nlz = buildLandingZone(edOrgId, desc, ingestionServer, path, userNames, preloadFiles);
            List<Map<String, Object>> newLandingZoneList = new ArrayList<Map<String, Object>>();
            newLandingZoneList.add(nlz);
            newTenant.put(LZ, newLandingZoneList);

            //In sandbox a user doesn't create a realm, so this is the only opportunity to create the custom roles
            if (isSandbox) {
                roleInitializer.dropAndBuildRoles(realmHelper.getSandboxRealmId());
            }
            return tenantService.create(newTenant);
        }
        // If more than exists, something is wrong
        if (existingIds.size() > 1) {
            throw new RuntimeException("Internal error: multiple tenant entry with identical IDs");
        }

        String existingTenantId = existingIds.get(0);
        // combine lzs from existing tenant and new tenant entry, overwriting with values of new
        // tenant entry if there is conflict.
        TreeSet allLandingZones = new TreeSet(new Comparator<Object>() {
            @Override
            public int compare(Object o1, Object o2) {
                Map<String, Object> lz1 = (Map<String, Object>) o1;
                Map<String, Object> lz2 = (Map<String, Object>) o2;
                if (!lz1.containsKey(LZ_EDUCATION_ORGANIZATION)
                        || !(lz1.get(LZ_EDUCATION_ORGANIZATION) instanceof String)) {
                    throw new RuntimeException("Badly formed tenant entry: " + lz1.toString());
                }
                if (!lz2.containsKey(LZ_EDUCATION_ORGANIZATION)
                        || !(lz2.get(LZ_EDUCATION_ORGANIZATION) instanceof String)) {
                    throw new RuntimeException("Badly formed tenant entry: " + lz2.toString());
                }
                return ((String) lz1.get(LZ_EDUCATION_ORGANIZATION)).compareTo((String) lz2
                        .get(LZ_EDUCATION_ORGANIZATION));
            }
        });

        Set<Map<String, Object>> all = allLandingZones;
        for (Map<String, Object> lz : all) {
            if (lz.get(LZ_EDUCATION_ORGANIZATION).equals(edOrgId)) {
                throw new TenantResourceCreationException(Status.CONFLICT,
                        "This tenant/educational organization combination all ready has a landing zone provisioned.");
            }
        }

        EntityBody existingBody = tenantService.get(existingTenantId);
        List existingLandingZones = (List) existingBody.get(LZ);
        allLandingZones.addAll(existingLandingZones);

        Map<String, Object> nlz = this.buildLandingZone(edOrgId, desc, ingestionServer, path, userNames, preloadFiles);
        allLandingZones.add(nlz);

        existingBody.put(LZ, new ArrayList(allLandingZones));
        tenantService.update(existingTenantId, existingBody);
        return existingTenantId;
    }

    private Map<String, Object> buildLandingZone(String edOrgId, String desc, String ingestionServer,
                                                 String path, List<String> userNames, List<String> preloadFiles) {
        Map<String, Object> newLandingZone = new HashMap<String, Object>();
        newLandingZone.put(LZ_EDUCATION_ORGANIZATION, edOrgId);
        newLandingZone.put(LZ_DESC, desc);
        newLandingZone.put(LZ_INGESTION_SERVER, ingestionServer);
        newLandingZone.put(LZ_PATH, path);
        newLandingZone.put(LZ_USER_NAMES, userNames);

        Map<String, Object> preload = new HashMap<String, Object>();
        if ((preloadFiles != null) && (!preloadFiles.isEmpty())) {
            preload.put(LZ_PRELOAD_FILES, preloadFiles);
            preload.put(LZ_PRELOAD_STATUS, LZ_PRELOAD_STATUS_READY);
        }
        newLandingZone.put(LZ_PRELOAD, preload);
        return newLandingZone;
    }

    /**
     * TODO: add javadoc
     *
     */
    static class MutableInt {
      int value = 0;
      public void increment() {
          ++value;
      }
      public int  get() {
          return value;
      }<commit_msg>fixed failure when preload is null<commit_after>/*
 * Copyright 2012 Shared Learning Collaborative, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package org.slc.sli.api.resources.security;

import java.io.File;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;

import javax.annotation.PostConstruct;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.UriInfo;

import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import org.slc.sli.api.config.EntityDefinitionStore;
import org.slc.sli.api.constants.ParameterConstants;
import org.slc.sli.api.init.RoleInitializer;
import org.slc.sli.api.representation.EntityBody;
import org.slc.sli.api.resources.Resource;
import org.slc.sli.api.resources.v1.DefaultCrudEndpoint;
import org.slc.sli.api.security.context.resolver.RealmHelper;
import org.slc.sli.api.service.EntityService;
import org.slc.sli.api.util.SecurityUtil;
import org.slc.sli.domain.NeutralCriteria;
import org.slc.sli.domain.NeutralQuery;
import org.slc.sli.domain.enums.Right;

/**
 *
 * Provides CRUD operations on registered application through the /tenants path.
 *
 * @author
 */
@Component
@Scope("request")
@Path("tenants")
@Produces({ Resource.JSON_MEDIA_TYPE + ";charset=utf-8" })
public class TenantResourceImpl extends DefaultCrudEndpoint implements TenantResource {

    @Autowired
    private EntityDefinitionStore store;

    @Autowired
    private RoleInitializer roleInitializer;

    @Value("${sli.tenant.landingZoneMountPoint}")
    private String landingZoneMountPoint;

    @Value("${sli.tenant.ingestionServers}")
    private String ingestionServers;

    @Autowired
    private RealmHelper realmHelper;

    private List<String> ingestionServerList;

    /* this is only available for unit testing */
    public void setIngestionServerList(List<String> testList) {
        ingestionServerList = testList;
    }

    //private Random random = new Random(System.currentTimeMillis());

    @PostConstruct
    protected void init() {
        ingestionServerList = Arrays.asList(ingestionServers.split(","));
    }

    public static final String UUID = "uuid";
    public static final String RESOURCE_NAME = "tenant";
    public static final String TENANT_ID = "tenantId";
    public static final String LZ = "landingZone";
    public static final String LZ_EDUCATION_ORGANIZATION = "educationOrganization";
    public static final String LZ_INGESTION_SERVER = "ingestionServer";
    public static final String LZ_PATH = "path";
    public static final String LZ_USER_NAMES = "userNames";
    public static final String LZ_DESC = "desc";
    public static final String LZ_INGESTION_SERVER_LOCALHOST = "localhost";
    public static final String LZ_PRELOAD              = "preload";
    public static final String LZ_PRELOAD_FILES        = "files";
    public static final String LZ_PRELOAD_STATUS       = "status";
    public static final String LZ_PRELOAD_STATUS_READY = "ready";


    @Autowired
    public TenantResourceImpl(EntityDefinitionStore entityDefs) {
        super(entityDefs, RESOURCE_NAME);
        store = entityDefs;
    }

    @Override
    @POST
    public Response create(EntityBody newTenant, @Context HttpHeaders headers, @Context final UriInfo uriInfo) {

        // Tenants can not be created using this class. They will be created via OnboardingResource
        return SecurityUtil.forbiddenResponse();
    }

    @Override
    public LandingZoneInfo createLandingZone(String tenantId, String edOrgId, List<String> preloadFiles, boolean isSandbox)
            throws TenantResourceCreationException {
        String newTenantId = createLandingZone(tenantId, edOrgId, null, null, preloadFiles, isSandbox);
        EntityService tenantService = store.lookupByResourceName(RESOURCE_NAME).getService();
        EntityBody newTenant = tenantService.get(newTenantId);
        @SuppressWarnings("unchecked")
        List<Map<String, Object>> newLzs = (List<Map<String, Object>>) newTenant.get(LZ);
        for (Map<String, Object> lz : newLzs) {
            if (edOrgId.equals(lz.get(LZ_EDUCATION_ORGANIZATION))) {
                return new LandingZoneInfo((String) lz.get(LZ_PATH), (String) lz.get(LZ_INGESTION_SERVER));
            }
        }
        throw new TenantResourceCreationException(Status.INTERNAL_SERVER_ERROR, "Failed to find landing zone information after creation.");
    }

    @SuppressWarnings({ "unchecked" })
    protected String createLandingZone(EntityBody newTenant, boolean isSandbox)
            throws TenantResourceCreationException {
        List<Map<String, Object>> newLzs = (List<Map<String, Object>>) newTenant.get(LZ);

        // NOTE: OnboardingResource may only send in one at a time
        if (1 != newLzs.size()) {
            throw new TenantResourceCreationException(Status.BAD_REQUEST,
                    "Only one landing zone may be provisioned at a time.  Please submit your requests individually.");
        }

        String tenantId = (String) newTenant.get(TENANT_ID);

        Map<String, Object> newLz = newLzs.get(0);
        String newEdOrg = (String) newLz.get(LZ_EDUCATION_ORGANIZATION);

        Map<String, Object> preload = (Map<String, Object>) newLz.get(LZ_PRELOAD);
        List<String> preloadFiles = (preload == null) ? (null) : (List<String>) preload.get(LZ_PRELOAD_FILES);

        return createLandingZone(tenantId, newEdOrg, (String) newLz.get(LZ_DESC),
                (List<String>) newLz.get(LZ_USER_NAMES), preloadFiles, isSandbox);
    }

    @SuppressWarnings({ "rawtypes", "unchecked" })
    protected String createLandingZone(final String tenantId, String edOrgId, String desc, List<String> userNames, List<String> preloadFiles, boolean isSandbox)
            throws TenantResourceCreationException {

        // get the exisint tenant resource
        EntityService tenantService = store.lookupByResourceName(RESOURCE_NAME).getService();
        NeutralQuery query = new NeutralQuery();
        query.addCriteria(new NeutralCriteria(TENANT_ID, "=", tenantId));

        String ingestionServer = findLeastLoadedIngestionServer();
        File inboundDirFile = new File(landingZoneMountPoint);
        File fullPath = new File(inboundDirFile, tenantId + "/" + DigestUtils.sha256Hex(edOrgId));
        String path = fullPath.getAbsolutePath();

        //resolve localhost ingestion server to the current server name
        if (ingestionServer.equals(LZ_INGESTION_SERVER_LOCALHOST)) {
            try {
                ingestionServer = InetAddress.getLocalHost().getHostName();
            } catch (UnknownHostException e) {
                throw new TenantResourceCreationException(Status.INTERNAL_SERVER_ERROR,
                        "Failed to resolve ingestion server for " + LZ_INGESTION_SERVER_LOCALHOST + ".");
            }
        }

        // look up ids of existing tenant entries
        List<String> existingIds = new ArrayList<String>();
        for (String id : tenantService.listIds(query)) {
            existingIds.add(id);
        }

        // If no tenant already exists, create one
        if (existingIds.size() == 0) {
            EntityBody newTenant = new EntityBody();
            newTenant.put(TENANT_ID, tenantId);
            Map<String, Object> nlz = buildLandingZone(edOrgId, desc, ingestionServer, path, userNames, preloadFiles);
            List<Map<String, Object>> newLandingZoneList = new ArrayList<Map<String, Object>>();
            newLandingZoneList.add(nlz);
            newTenant.put(LZ, newLandingZoneList);

            //In sandbox a user doesn't create a realm, so this is the only opportunity to create the custom roles
            if (isSandbox) {
                roleInitializer.dropAndBuildRoles(realmHelper.getSandboxRealmId());
            }
            return tenantService.create(newTenant);
        }
        // If more than exists, something is wrong
        if (existingIds.size() > 1) {
            throw new RuntimeException("Internal error: multiple tenant entry with identical IDs");
        }

        String existingTenantId = existingIds.get(0);
        // combine lzs from existing tenant and new tenant entry, overwriting with values of new
        // tenant entry if there is conflict.
        TreeSet allLandingZones = new TreeSet(new Comparator<Object>() {
            @Override
            public int compare(Object o1, Object o2) {
                Map<String, Object> lz1 = (Map<String, Object>) o1;
                Map<String, Object> lz2 = (Map<String, Object>) o2;
                if (!lz1.containsKey(LZ_EDUCATION_ORGANIZATION)
                        || !(lz1.get(LZ_EDUCATION_ORGANIZATION) instanceof String)) {
                    throw new RuntimeException("Badly formed tenant entry: " + lz1.toString());
                }
                if (!lz2.containsKey(LZ_EDUCATION_ORGANIZATION)
                        || !(lz2.get(LZ_EDUCATION_ORGANIZATION) instanceof String)) {
                    throw new RuntimeException("Badly formed tenant entry: " + lz2.toString());
                }
                return ((String) lz1.get(LZ_EDUCATION_ORGANIZATION)).compareTo((String) lz2
                        .get(LZ_EDUCATION_ORGANIZATION));
            }
        });

        Set<Map<String, Object>> all = allLandingZones;
        for (Map<String, Object> lz : all) {
            if (lz.get(LZ_EDUCATION_ORGANIZATION).equals(edOrgId)) {
                throw new TenantResourceCreationException(Status.CONFLICT,
                        "This tenant/educational organization combination all ready has a landing zone provisioned.");
            }
        }

        EntityBody existingBody = tenantService.get(existingTenantId);
        List existingLandingZones = (List) existingBody.get(LZ);
        allLandingZones.addAll(existingLandingZones);

        Map<String, Object> nlz = this.buildLandingZone(edOrgId, desc, ingestionServer, path, userNames, preloadFiles);
        allLandingZones.add(nlz);

        existingBody.put(LZ, new ArrayList(allLandingZones));
        tenantService.update(existingTenantId, existingBody);
        return existingTenantId;
    }

    private Map<String, Object> buildLandingZone(String edOrgId, String desc, String ingestionServer,
                                                 String path, List<String> userNames, List<String> preloadFiles) {
        Map<String, Object> newLandingZone = new HashMap<String, Object>();
        newLandingZone.put(LZ_EDUCATION_ORGANIZATION, edOrgId);
        newLandingZone.put(LZ_DESC, desc);
        newLandingZone.put(LZ_INGESTION_SERVER, ingestionServer);
        newLandingZone.put(LZ_PATH, path);
        newLandingZone.put(LZ_USER_NAMES, userNames);

        if ((preloadFiles != null) && (!preloadFiles.isEmpty())) {
            Map<String, Object> preload = new HashMap<String, Object>();
            preload.put(LZ_PRELOAD_FILES, preloadFiles);
            preload.put(LZ_PRELOAD_STATUS, LZ_PRELOAD_STATUS_READY);
            newLandingZone.put(LZ_PRELOAD, preload);
        }
        return newLandingZone;
    }

    /**
     * TODO: add javadoc
     *
     */
    static class MutableInt {
      int value = 0;
      public void increment() {
          ++value;
      }
      public int  get() {
          return value;
      }
{'fraction_non_alphanumeric': 0.09499542142771827, 'fraction_numerical': 0.002627702352988016, 'mean_word_length': 3.116355293346444, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3231754', 'n_tokens_mistral': 7391, 'n_tokens_neox': 7183, 'n_words': 1747}