<commit_before>package de.ids_mannheim.korap.resource.rewrite;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import de.ids_mannheim.korap.utils.JsonUtils;

import java.util.Iterator;

/**
 * @author hanl
 * @date 28/07/2015
 */
//todo: test
public class CollectionCleanupFilter extends RewriteQuery {

    // track path to operand
    private StringBuilder builder;

    public CollectionCleanupFilter() {
        this.builder = new StringBuilder();
    }

    @Override
    public JsonNode rewrite(KoralNode node) {
        if (node.rawNode().has("collection")) {
            JsonNode coll = node.rawNode().path("collection");
            process(coll);
        }
        return null;
    }

    private JsonNode process(JsonNode root) {
        if (root.isObject()) {
            if (root.has("operands")) {
                JsonNode node = root.at("/operands");
                Iterator<JsonNode> it = node.elements();
                while (it.hasNext()) {
                    JsonNode n = it.next();
                    JsonNode s = process(n);
                    if (s == null)
                        it.remove();
                }
            }

            JsonNode sub = processNodes(root);
            if (!root.equals(sub)) {
                if (sub == null) {
                    //can't do anything here -- fixme: edge case?!
                    return null;
                }else if (sub.isObject()) {
                    ObjectNode ob = (ObjectNode) root;
                    ob.removeAll();
                    ob.putAll((ObjectNode) sub);
                }

            }
        }
        return root;
    }

    // return null deletes node, if node return replace at level -1
    private JsonNode processNodes(JsonNode jsonNode) {
        if (jsonNode.isObject()) {
            if (jsonNode.has("operands")) {
                JsonNode node = jsonNode.at("/operands");
                int count = node.size();
                if (count == 1) {
                    // move to super node if any
                    return node.path(0);
                }else if (count == 0) {
                    // remove container
                    return null;
                }
                return jsonNode;
            }
        }
        return JsonUtils.createArrayNode();
    }
}
<commit_msg>rewrite classes<commit_after>package de.ids_mannheim.korap.resource.rewrite;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import de.ids_mannheim.korap.utils.JsonUtils;

import java.util.Iterator;

/**
 * @author hanl
 * @date 28/07/2015
 */
public class CollectionCleanupFilter extends RewriteQuery {

    // track path to operand
    @Deprecated
    private StringBuilder builder;

    public CollectionCleanupFilter() {
        this.builder = new StringBuilder();
    }

    @Override
    public JsonNode rewrite(KoralNode node) {
        if (node.rawNode().has("collection")) {
            JsonNode coll = node.rawNode().path("collection");
            process(coll);
        }
        return null;
    }

    private JsonNode process(JsonNode root) {
        JsonNode sub = root;
        if (root.isObject()) {
            if (root.has("operands")) {
                JsonNode node = root.at("/operands");
                Iterator<JsonNode> it = node.elements();
                while (it.hasNext()) {
                    JsonNode n = it.next();
                    JsonNode s = process(n);
                    if (s == null)
                        it.remove();
                }

                int count = node.size();
                if (count == 1)
                    sub = node.path(0);
                else if (count
                        == 0) // can't do anything here -- fixme: edge case?!
                    return null;
            }

            if (!root.equals(sub)) {
                if (sub.isObject()) {
                    ObjectNode ob = (ObjectNode) root;
                    ob.removeAll();
                    ob.putAll((ObjectNode) sub);
                }
            }
        }
        return root;
    }

    // return null deletes node, if node return replace at level -1
    @Deprecated
    private JsonNode processNodes(JsonNode jsonNode) {
        if (jsonNode.isObject()) {
            if (jsonNode.has("operands")) {
                JsonNode node = jsonNode.at("/operands");
                int count = node.size();
                if (count == 1) {
                    // move to super node if any
                    return node.path(0);
                }else if (count == 0) {
                    // remove container
                    return null;
                }
                return jsonNode;
            }
        }
        return JsonUtils.createArrayNode();
    }
}

{'fraction_non_alphanumeric': 0.10799091472227958, 'fraction_numerical': 0.005575056782985753, 'mean_word_length': 1.371023005384239, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3113327', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1216, 'n_words': 307}