<commit_before>/*
 * Copyright 2019 Flipkart Internet, pvt ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.flipkart.poseidon.oas;

import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.JavaType;
import com.flipkart.poseidon.api.APIManager;
import com.flipkart.poseidon.helper.ClassPathHelper;
import com.flipkart.poseidon.legoset.PoseidonLegoSet;
import com.flipkart.poseidon.model.oas.Response;
import com.flipkart.poseidon.model.oas.Responses;
import com.flipkart.poseidon.pojos.EndpointPOJO;
import com.flipkart.poseidon.pojos.ParamPOJO;
import com.flipkart.poseidon.pojos.ParamsPOJO;
import com.flipkart.poseidon.pojos.TaskPOJO;
import com.flipkart.poseidon.utils.ApiHelper;
import com.flipkart.poseidon.validator.BlocksValidator;
import com.google.common.reflect.ClassPath;
import com.google.common.reflect.TypeToken;
import flipkart.lego.api.entities.DataSource;
import io.swagger.v3.core.util.Json;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.media.*;
import io.swagger.v3.oas.models.parameters.Parameter;
import io.swagger.v3.oas.models.parameters.RequestBody;
import io.swagger.v3.oas.models.responses.ApiResponse;
import io.swagger.v3.oas.models.responses.ApiResponses;
import org.springframework.http.HttpMethod;

import javax.annotation.Nonnull;
import javax.validation.constraints.NotNull;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;

import static com.flipkart.poseidon.helpers.ObjectMapperHelper.getMapper;

/**
 * Created by shrey.garg on 2019-01-01.
 */
public class SchemaGenerator {
    private static final Map<String, JavaType> modelTypes = new HashMap<>();
    private static final Set<String> globalModelSet = new HashSet<>();
    private static final Map<String, Class<? extends DataSource<?>>> datasources = new HashMap<>();
    private static final Set<String> noBuilderClasses = new HashSet<>();

    private static final List<Class<? extends Annotation>> nonNullAnnotations = Arrays.asList(
            NotNull.class,
            Nonnull.class
    );

    public static void main(String[] args) throws IOException, NoSuchMethodException {
        if (args.length < 3) {
            throw new UnsupportedOperationException("Correct Usage: <api-dir> <schema-target> <datasource-packages-csv>");
        }

        final List<String> validConfigs = new ArrayList<>();
        List<EndpointPOJO> pojos = new ArrayList<>();

        final Path dir = Paths.get(args[0]);
        final Path genDir = Paths.get(args[1]);
        final Path apiDir = genDir.resolve("apis/v1");
        final Path apiFile = apiDir.resolve("ServiceResource.json");
        final Path modelsDir = genDir.resolve("models");

        if (Files.exists(modelsDir)) {
            Files.walk(modelsDir)
                    .sorted(Comparator.reverseOrder())
                    .peek(System.out::println)
                    .forEach(deleteIfExists);
        }

        if (args.length == 4) {
            noBuilderClasses.addAll(Arrays.asList(args[3].split(",")));
        }

        try {
            APIManager.scanAndAdd(dir, validConfigs);
            for (String config : validConfigs) {
                pojos.add(getMapper().readValue(config, EndpointPOJO.class));
            }
            pojos = pojos.stream().sorted(Comparator.comparing(EndpointPOJO::getName)).collect(Collectors.toList());
        } catch (Exception e) {
            throw new UnsupportedOperationException("Error while reading EndpointPOJO", e);
        }

        final String[] datasourcePackages = args[2].split(",");
        BlocksValidator.main(datasourcePackages);

        fetchDataSources(datasourcePackages);

        OpenAPI openAPI = new OpenAPI();
        Info info = new Info();
        info.title("ServiceResource");
        info.version("1.0");
        openAPI.info(info);

        final Map<String, Map<PathItem.HttpMethod, Operation>> paths = new LinkedHashMap<>();
        for (EndpointPOJO endpointPOJO : pojos) {
            final Operation operation = new Operation();
            final ParamsPOJO paramsPOJO = endpointPOJO.getParams();
            if (paramsPOJO != null) {
                if (paramsPOJO.getRequired() != null) {
                    for (ParamPOJO paramPOJO : paramsPOJO.getRequired()) {
                        if (paramPOJO.isFile()) {
                            continue;
                        }

                        if (paramPOJO.isBody()) {
                            final RequestBody requestBody = new RequestBody();
                            requestBody.required(true);
                            final Content content = new Content();
                            final MediaType mediaType = new MediaType();
                            mediaType.schema(createSchema(paramPOJO));
                            content.addMediaType("application/json", mediaType);
                            requestBody.content(content);
                            operation.requestBody(requestBody);
                            continue;
                        }

                        operation.addParametersItem(createParameter(paramPOJO).required(true));
                    }
                }

                if (paramsPOJO.getOptional() != null) {
                    for (ParamPOJO paramPOJO : paramsPOJO.getOptional()) {
                        if (paramPOJO.isFile() || paramPOJO.isBody()) {
                            continue;
                        }

                        operation.addParametersItem(createParameter(paramPOJO));
                    }
                }
            }

            ApiResponses responses = new ApiResponses();
            if (endpointPOJO.getResponse() instanceof String) {
                String responseString = (String) endpointPOJO.getResponse();
                responseString = responseString.substring(3, responseString.length() - 2);
                if (responseString.startsWith("$")) {
                    responseString = responseString.substring(1);
                }

                final TaskPOJO taskPOJO = endpointPOJO.getTasks().get(responseString);
                final Class<? extends DataSource<?>> dsClass = datasources.get(taskPOJO.getName());
                final Type resolvedType = TypeToken.of(dsClass)
                        .resolveType(dsClass.getMethod("call").getGenericReturnType())
                        .getType();
                resolveAPIResponse("200", resolvedType, responses, modelsDir);
                handleOtherResponses(dsClass, responses, modelsDir);
            }

            operation.responses(responses);
            operation.operationId(endpointPOJO.getName());
            paths.computeIfAbsent(endpointPOJO.getUrl(), s -> new LinkedHashMap<>()).put(getHttpMethod(endpointPOJO.getHttpMethod()), operation);
        }

        paths.forEach((k, v) -> {
            final PathItem pathItem = new PathItem();
            pathItem.addExtension("x-old-path", k);
            v.forEach(pathItem::operation);
            openAPI.path(k, pathItem);
        });

        final byte[] apiBytes = Json.mapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(openAPI);
        Files.createDirectories(apiFile.getParent());
        Files.write(apiFile, apiBytes, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);

        modelTypes.forEach((k, v) -> {
            try {
                final Path modelFile = modelsDir.resolve(k + ".json");
                Files.createDirectories(modelFile.getParent());
                Files.write(modelFile, Json.mapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(generateSchema(v, modelsDir)), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
            } catch (IOException e) {
                throw new RuntimeException("Error while create model files", e);
            }
        });
    }

    private static void resolveAPIResponse(String status, Type resolvedType, ApiResponses responses, Path modelsDir) {
        final Schema<?> responseSchema = processType(resolvedType, modelsDir);
        final ApiResponse response = new ApiResponse();
        final Content content = new Content();
        final MediaType mediaType = new MediaType();
        mediaType.schema(responseSchema);
        content.addMediaType("application/json", mediaType);
        response.content(content);
        response.description("Some random thing");
        responses.addApiResponse(status, response);
    }

    private static void handleOtherResponses(Class<? extends DataSource<?>> dsClass, ApiResponses responses, Path modelsDir) {
        final Responses responsesAnnotation = dsClass.getAnnotation(Responses.class);
        if (responsesAnnotation == null) {
            return;
        }

        final Response[] responseArray = responsesAnnotation.value();
        for (Response response : responseArray) {
            resolveAPIResponse(String.valueOf(response.status()), response.responseClass(), responses, modelsDir);
        }
    }

    private static Schema<?> generateSchema(JavaType javaType, Path modelsDir) {
        final Class<?> rawClass = javaType.getRawClass();
        return processClass(rawClass, modelsDir);
    }

    private static Schema<?> processClass(Class<?> clazz, Path modelsDir) {
        final Map<String, Class<?>> referencedClasses = new HashMap<>();
        final int globalSetSize = globalModelSet.size();

        if (clazz.isEnum()) {
            final Enum<?>[] enumConstants = ((Class<? extends Enum<?>>) clazz).getEnumConstants();
            StringSchema schema = new StringSchema();
            Arrays.stream(enumConstants).forEach(e -> schema.addEnumItem(e.name()));
            return schema;
        }

        ComposedSchema schema = new ComposedSchema();
        schema.type("object");

        if (!noBuilderClasses.contains(clazz.getName())) {
            schema.addExtension("x-create-builder", true);
        }

        final Class<?> superclass = clazz.getSuperclass();
        if (superclass != null && superclass != Object.class) {
            schema.allOf(Collections.singletonList(createReference(superclass, clazz, referencedClasses)));
        }

        handleJsonSubTypes(clazz, schema, referencedClasses);

        final Field[] declaredFields = clazz.getDeclaredFields();
        for (Field field : declaredFields) {
            if (Modifier.isStatic(field.getModifiers())) {
                continue;
            }

            schema.addProperties(field.getName(), processField(clazz, field.getType(), field.getGenericType(), referencedClasses));
            if (isRequiredProperty(field)) {
                schema.addRequiredItem(field.getName());
            }
        }

        if (globalSetSize < globalModelSet.size()) {
            referencedClasses.forEach((k, v) -> {
                try {
                    final Path modelFile = modelsDir.resolve(k + ".json");
                    Files.createDirectories(modelFile.getParent());
                    final Schema<?> referencedSchema = processClass(v, modelsDir);
                    Files.write(modelFile, Json.mapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(referencedSchema), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
                } catch (IOException e) {
                    throw new RuntimeException("Error while create model files", e);
                }
            });
        }

        return schema;
    }

    private static void handleJsonSubTypes(Class<?> clazz, ComposedSchema schema, Map<String, Class<?>> referencedClasses) {
        final JsonTypeInfo typeInfo = clazz.getAnnotation(JsonTypeInfo.class);
        final JsonSubTypes subTypes = clazz.getAnnotation(JsonSubTypes.class);

        if (typeInfo != null && subTypes != null) {
            schema.discriminator(new Discriminator().propertyName(typeInfo.property()));

            for (JsonSubTypes.Type type : subTypes.value()) {
                schema.addOneOfItem(createReference(type.value(), clazz, referencedClasses));
            }
        }
    }

    private static boolean isRequiredProperty(Field field) {
        for (Class<? extends Annotation> nonNullAnnotation : nonNullAnnotations) {<commit_msg>making schema generation non relative again (temp)<commit_after>/*
 * Copyright 2019 <NAME>, pvt ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.flipkart.poseidon.oas;

import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.JavaType;
import com.flipkart.poseidon.api.APIManager;
import com.flipkart.poseidon.helper.ClassPathHelper;
import com.flipkart.poseidon.legoset.PoseidonLegoSet;
import com.flipkart.poseidon.model.oas.Response;
import com.flipkart.poseidon.model.oas.Responses;
import com.flipkart.poseidon.pojos.EndpointPOJO;
import com.flipkart.poseidon.pojos.ParamPOJO;
import com.flipkart.poseidon.pojos.ParamsPOJO;
import com.flipkart.poseidon.pojos.TaskPOJO;
import com.flipkart.poseidon.utils.ApiHelper;
import com.flipkart.poseidon.validator.BlocksValidator;
import com.google.common.reflect.ClassPath;
import com.google.common.reflect.TypeToken;
import flipkart.lego.api.entities.DataSource;
import io.swagger.v3.core.util.Json;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.media.*;
import io.swagger.v3.oas.models.parameters.Parameter;
import io.swagger.v3.oas.models.parameters.RequestBody;
import io.swagger.v3.oas.models.responses.ApiResponse;
import io.swagger.v3.oas.models.responses.ApiResponses;
import org.springframework.http.HttpMethod;

import javax.annotation.Nonnull;
import javax.validation.constraints.NotNull;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;

import static com.flipkart.poseidon.helpers.ObjectMapperHelper.getMapper;

/**
 * Created by shrey.garg on 2019-01-01.
 */
public class SchemaGenerator {
    private static final Map<String, JavaType> modelTypes = new HashMap<>();
    private static final Set<String> globalModelSet = new HashSet<>();
    private static final Map<String, Class<? extends DataSource<?>>> datasources = new HashMap<>();
    private static final Set<String> noBuilderClasses = new HashSet<>();

    private static final List<Class<? extends Annotation>> nonNullAnnotations = Arrays.asList(
            NotNull.class,
            Nonnull.class
    );

    public static void main(String[] args) throws IOException, NoSuchMethodException {
        if (args.length < 3) {
            throw new UnsupportedOperationException("Correct Usage: <api-dir> <schema-target> <datasource-packages-csv>");
        }

        final List<String> validConfigs = new ArrayList<>();
        List<EndpointPOJO> pojos = new ArrayList<>();

        final Path dir = Paths.get(args[0]);
        final Path genDir = Paths.get(args[1]);
        final Path apiDir = genDir.resolve("apis/v1");
        final Path apiFile = apiDir.resolve("ServiceResource.json");
        final Path modelsDir = genDir.resolve("models");

        if (Files.exists(modelsDir)) {
            Files.walk(modelsDir)
                    .sorted(Comparator.reverseOrder())
                    .peek(System.out::println)
                    .forEach(deleteIfExists);
        }

        if (args.length == 4) {
            noBuilderClasses.addAll(Arrays.asList(args[3].split(",")));
        }

        try {
            APIManager.scanAndAdd(dir, validConfigs);
            for (String config : validConfigs) {
                pojos.add(getMapper().readValue(config, EndpointPOJO.class));
            }
            pojos = pojos.stream().sorted(Comparator.comparing(EndpointPOJO::getName)).collect(Collectors.toList());
        } catch (Exception e) {
            throw new UnsupportedOperationException("Error while reading EndpointPOJO", e);
        }

        final String[] datasourcePackages = args[2].split(",");
        BlocksValidator.main(datasourcePackages);

        fetchDataSources(datasourcePackages);

        OpenAPI openAPI = new OpenAPI();
        Info info = new Info();
        info.title("ServiceResource");
        info.version("1.0");
        openAPI.info(info);

        final Map<String, Map<PathItem.HttpMethod, Operation>> paths = new LinkedHashMap<>();
        for (EndpointPOJO endpointPOJO : pojos) {
            final Operation operation = new Operation();
            final ParamsPOJO paramsPOJO = endpointPOJO.getParams();
            if (paramsPOJO != null) {
                if (paramsPOJO.getRequired() != null) {
                    for (ParamPOJO paramPOJO : paramsPOJO.getRequired()) {
                        if (paramPOJO.isFile()) {
                            continue;
                        }

                        if (paramPOJO.isBody()) {
                            final RequestBody requestBody = new RequestBody();
                            requestBody.required(true);
                            final Content content = new Content();
                            final MediaType mediaType = new MediaType();
                            mediaType.schema(createSchema(paramPOJO));
                            content.addMediaType("application/json", mediaType);
                            requestBody.content(content);
                            operation.requestBody(requestBody);
                            continue;
                        }

                        operation.addParametersItem(createParameter(paramPOJO).required(true));
                    }
                }

                if (paramsPOJO.getOptional() != null) {
                    for (ParamPOJO paramPOJO : paramsPOJO.getOptional()) {
                        if (paramPOJO.isFile() || paramPOJO.isBody()) {
                            continue;
                        }

                        operation.addParametersItem(createParameter(paramPOJO));
                    }
                }
            }

            ApiResponses responses = new ApiResponses();
            if (endpointPOJO.getResponse() instanceof String) {
                String responseString = (String) endpointPOJO.getResponse();
                responseString = responseString.substring(3, responseString.length() - 2);
                if (responseString.startsWith("$")) {
                    responseString = responseString.substring(1);
                }

                final TaskPOJO taskPOJO = endpointPOJO.getTasks().get(responseString);
                final Class<? extends DataSource<?>> dsClass = datasources.get(taskPOJO.getName());
                final Type resolvedType = TypeToken.of(dsClass)
                        .resolveType(dsClass.getMethod("call").getGenericReturnType())
                        .getType();
                resolveAPIResponse("200", resolvedType, responses, modelsDir);
                handleOtherResponses(dsClass, responses, modelsDir);
            }

            operation.responses(responses);
            operation.operationId(endpointPOJO.getName());
            paths.computeIfAbsent(endpointPOJO.getUrl(), s -> new LinkedHashMap<>()).put(getHttpMethod(endpointPOJO.getHttpMethod()), operation);
        }

        paths.forEach((k, v) -> {
            final PathItem pathItem = new PathItem();
            pathItem.addExtension("x-old-path", k);
            v.forEach(pathItem::operation);
            openAPI.path(k, pathItem);
        });

        final byte[] apiBytes = Json.mapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(openAPI);
        Files.createDirectories(apiFile.getParent());
        Files.write(apiFile, apiBytes, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);

        modelTypes.forEach((k, v) -> {
            try {
                final Path modelFile = modelsDir.resolve(k + ".json");
                Files.createDirectories(modelFile.getParent());
                Files.write(modelFile, Json.mapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(generateSchema(v, modelsDir)), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
            } catch (IOException e) {
                throw new RuntimeException("Error while create model files", e);
            }
        });
    }

    private static void resolveAPIResponse(String status, Type resolvedType, ApiResponses responses, Path modelsDir) {
        final Schema<?> responseSchema = processType(resolvedType, modelsDir);
        final ApiResponse response = new ApiResponse();
        final Content content = new Content();
        final MediaType mediaType = new MediaType();
        mediaType.schema(responseSchema);
        content.addMediaType("application/json", mediaType);
        response.content(content);
        response.description("Some random thing");
        responses.addApiResponse(status, response);
    }

    private static void handleOtherResponses(Class<? extends DataSource<?>> dsClass, ApiResponses responses, Path modelsDir) {
        final Responses responsesAnnotation = dsClass.getAnnotation(Responses.class);
        if (responsesAnnotation == null) {
            return;
        }

        final Response[] responseArray = responsesAnnotation.value();
        for (Response response : responseArray) {
            resolveAPIResponse(String.valueOf(response.status()), response.responseClass(), responses, modelsDir);
        }
    }

    private static Schema<?> generateSchema(JavaType javaType, Path modelsDir) {
        final Class<?> rawClass = javaType.getRawClass();
        return processClass(rawClass, modelsDir);
    }

    private static Schema<?> processClass(Class<?> clazz, Path modelsDir) {
        final Map<String, Class<?>> referencedClasses = new HashMap<>();
        final int globalSetSize = globalModelSet.size();

        if (clazz.isEnum()) {
            final Enum<?>[] enumConstants = ((Class<? extends Enum<?>>) clazz).getEnumConstants();
            StringSchema schema = new StringSchema();
            Arrays.stream(enumConstants).forEach(e -> schema.addEnumItem(e.name()));
            return schema;
        }

        ComposedSchema schema = new ComposedSchema();
        schema.type("object");

        if (!noBuilderClasses.contains(clazz.getName())) {
            schema.addExtension("x-create-builder", true);
        }

        final Class<?> superclass = clazz.getSuperclass();
        if (superclass != null && superclass != Object.class) {
            schema.allOf(Collections.singletonList(createReference(superclass, null, referencedClasses)));
        }

        handleJsonSubTypes(clazz, schema, referencedClasses);

        final Field[] declaredFields = clazz.getDeclaredFields();
        for (Field field : declaredFields) {
            if (Modifier.isStatic(field.getModifiers())) {
                continue;
            }

            schema.addProperties(field.getName(), processField(clazz, field.getType(), field.getGenericType(), referencedClasses));
            if (isRequiredProperty(field)) {
                schema.addRequiredItem(field.getName());
            }
        }

        if (globalSetSize < globalModelSet.size()) {
            referencedClasses.forEach((k, v) -> {
                try {
                    final Path modelFile = modelsDir.resolve(k + ".json");
                    Files.createDirectories(modelFile.getParent());
                    final Schema<?> referencedSchema = processClass(v, modelsDir);
                    Files.write(modelFile, Json.mapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(referencedSchema), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
                } catch (IOException e) {
                    throw new RuntimeException("Error while create model files", e);
                }
            });
        }

        return schema;
    }

    private static void handleJsonSubTypes(Class<?> clazz, ComposedSchema schema, Map<String, Class<?>> referencedClasses) {
        final JsonTypeInfo typeInfo = clazz.getAnnotation(JsonTypeInfo.class);
        final JsonSubTypes subTypes = clazz.getAnnotation(JsonSubTypes.class);

        if (typeInfo != null && subTypes != null) {
            schema.discriminator(new Discriminator().propertyName(typeInfo.property()));

            for (JsonSubTypes.Type type : subTypes.value()) {
                schema.addOneOfItem(createReference(type.value(), null, referencedClasses));
            }
        }
    }

    private static boolean isRequiredProperty(Field field) {
        for (Class<? extends Annotation> nonNullAnnotation : nonNullAnnotations) {
{'fraction_non_alphanumeric': 0.10826638636534275, 'fraction_numerical': 0.003078309182371049, 'mean_word_length': 2.555659369994661, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 102, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7320439', 'n_tokens_mistral': 7062, 'n_tokens_neox': 6834, 'n_words': 1457}