<commit_before>package de.zalando.aruha.nakadi.validation;

import de.zalando.aruha.nakadi.domain.EventType;
import de.zalando.aruha.nakadi.domain.ValidationStrategyConfiguration;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

public class EventValidation {
    public static EventTypeValidator forType(final EventType eventType) {
        final EventTypeValidator etv = new EventTypeValidator(eventType);
        final ValidationStrategyConfiguration vsc = new ValidationStrategyConfiguration();
        vsc.setStrategyName(EventBodyMustRespectSchema.NAME);

        return etv.withConfiguration(vsc);
    }

    public static JSONObject effectiveSchema(final EventType eventType) throws JSONException {
        final JSONObject schema = new JSONObject(eventType.getSchema().getSchema());

        switch (eventType.getCategory()) {
            case BUSINESS: return addMetadata(schema, eventType);
            case DATA: return wrapSchemaInData(schema, eventType);
            default: return schema;
        }
    }

    private static JSONObject wrapSchemaInData(final JSONObject schema, EventType eventType) {
        final JSONObject wrapper = new JSONObject();

        normalizeSchema(wrapper);

        addMetadata(wrapper, eventType);

        wrapper.getJSONObject("properties").put("data_type", new JSONObject("{\"type\": \"string\"}"));

        wrapper.getJSONObject("properties").put("data_op", new JSONObject("{\"type\": \"string\", \"enum\": [\"C\", \"U\", \"D\", \"S\"]}"));

        wrapper.getJSONObject("properties").put("data", schema);

        wrapper.put("additionalProperties", false);

        addToRequired(wrapper, new String[]{ "data_type", "data_op", "data" });

        return wrapper;
    }

    private static JSONObject addMetadata(final JSONObject schema, EventType eventType) {
        normalizeSchema(schema);

        final JSONObject metadata = new JSONObject();
        final JSONObject metadataProperties = new JSONObject();

        final JSONObject uuid = new JSONObject("{\"type\": \"string\", \"pattern\": \"^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$\"}");
        final JSONObject arrayOfUUIDs = new JSONObject();
        arrayOfUUIDs.put("type", "array");
        arrayOfUUIDs.put("items", uuid);
        final JSONObject eventTypeString = new JSONObject("{\"type\": \"string\", \"enum\": [\"" + eventType.getName() + "\"]}");
        final JSONObject string = new JSONObject("{\"type\": \"string\"}");

        final String dateTimePattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T| )[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$";
        final JSONObject dateTime = new JSONObject("{\"type\": \"string\", \"pattern\": \"" + dateTimePattern + "\"}");

        metadataProperties.put("eid", uuid);
        metadataProperties.put("event_type", eventTypeString);
        metadataProperties.put("occurred_at", dateTime);
        metadataProperties.put("parent_eids", uuid);
        metadataProperties.put("root_eid", uuid);
        metadataProperties.put("flow_id", string);

        metadata.put("type", "object");
        metadata.put("properties", metadataProperties);
        metadata.put("required", Arrays.asList(new String[]{"eid", "occurred_at"}));
        metadata.put("additionalProperties", false);

        schema.getJSONObject("properties").put("metadata", metadata);

        addToRequired(schema, new String[]{ "metadata" });

        return schema;
    }

    private static void addToRequired(final JSONObject schema, final String[] toBeRequired) {
        final Set<String> required = new HashSet<>(Arrays.asList(toBeRequired));

        JSONArray currentRequired = schema.getJSONArray("required");

        for(int i = 0; i < currentRequired.length(); i++) {
            required.add(currentRequired.getString(i));
        }

        schema.put("required", required);
    }

    private static void normalizeSchema(final JSONObject schema) {
        schema.put("type", "object");

        if (!schema.has("properties")) {
            schema.put("properties", new JSONObject());
        }

        if (!schema.has("required")) {
            schema.put("required", new JSONArray());
        }
    }
}

<commit_msg>aruha-114 remove root_eid since it was removed from the api<commit_after>package de.zalando.aruha.nakadi.validation;

import de.zalando.aruha.nakadi.domain.EventType;
import de.zalando.aruha.nakadi.domain.ValidationStrategyConfiguration;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

public class EventValidation {
    public static EventTypeValidator forType(final EventType eventType) {
        final EventTypeValidator etv = new EventTypeValidator(eventType);
        final ValidationStrategyConfiguration vsc = new ValidationStrategyConfiguration();
        vsc.setStrategyName(EventBodyMustRespectSchema.NAME);

        return etv.withConfiguration(vsc);
    }

    public static JSONObject effectiveSchema(final EventType eventType) throws JSONException {
        final JSONObject schema = new JSONObject(eventType.getSchema().getSchema());

        switch (eventType.getCategory()) {
            case BUSINESS: return addMetadata(schema, eventType);
            case DATA: return wrapSchemaInData(schema, eventType);
            default: return schema;
        }
    }

    private static JSONObject wrapSchemaInData(final JSONObject schema, EventType eventType) {
        final JSONObject wrapper = new JSONObject();

        normalizeSchema(wrapper);

        addMetadata(wrapper, eventType);

        wrapper.getJSONObject("properties").put("data_type", new JSONObject("{\"type\": \"string\"}"));

        wrapper.getJSONObject("properties").put("data_op", new JSONObject("{\"type\": \"string\", \"enum\": [\"C\", \"U\", \"D\", \"S\"]}"));

        wrapper.getJSONObject("properties").put("data", schema);

        wrapper.put("additionalProperties", false);

        addToRequired(wrapper, new String[]{ "data_type", "data_op", "data" });

        return wrapper;
    }

    private static JSONObject addMetadata(final JSONObject schema, EventType eventType) {
        normalizeSchema(schema);

        final JSONObject metadata = new JSONObject();
        final JSONObject metadataProperties = new JSONObject();

        final JSONObject uuid = new JSONObject("{\"type\": \"string\", \"pattern\": \"^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$\"}");
        final JSONObject arrayOfUUIDs = new JSONObject();
        arrayOfUUIDs.put("type", "array");
        arrayOfUUIDs.put("items", uuid);
        final JSONObject eventTypeString = new JSONObject("{\"type\": \"string\", \"enum\": [\"" + eventType.getName() + "\"]}");
        final JSONObject string = new JSONObject("{\"type\": \"string\"}");

        final String dateTimePattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T| )[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$";
        final JSONObject dateTime = new JSONObject("{\"type\": \"string\", \"pattern\": \"" + dateTimePattern + "\"}");

        metadataProperties.put("eid", uuid);
        metadataProperties.put("event_type", eventTypeString);
        metadataProperties.put("occurred_at", dateTime);
        metadataProperties.put("parent_eids", uuid);
        metadataProperties.put("flow_id", string);

        metadata.put("type", "object");
        metadata.put("properties", metadataProperties);
        metadata.put("required", Arrays.asList(new String[]{"eid", "occurred_at"}));
        metadata.put("additionalProperties", false);

        schema.getJSONObject("properties").put("metadata", metadata);

        addToRequired(schema, new String[]{ "metadata" });

        return schema;
    }

    private static void addToRequired(final JSONObject schema, final String[] toBeRequired) {
        final Set<String> required = new HashSet<>(Arrays.asList(toBeRequired));

        JSONArray currentRequired = schema.getJSONArray("required");

        for(int i = 0; i < currentRequired.length(); i++) {
            required.add(currentRequired.getString(i));
        }

        schema.put("required", required);
    }

    private static void normalizeSchema(final JSONObject schema) {
        schema.put("type", "object");

        if (!schema.has("properties")) {
            schema.put("properties", new JSONObject());
        }

        if (!schema.has("required")) {
            schema.put("required", new JSONArray());
        }
    }
}


{'fraction_non_alphanumeric': 0.1595106186518929, 'fraction_numerical': 0.010272391505078486, 'mean_word_length': 3.3762626262626263, 'pattern_counts': {'":': 20, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1891468', 'n_tokens_mistral': 2502, 'n_tokens_neox': 2431, 'n_words': 499}