<commit_before>package de.ids_mannheim.korap.query.serialize;

import de.ids_mannheim.korap.query.PoliqarpPlusLexer;
import de.ids_mannheim.korap.query.PoliqarpPlusParser;
import de.ids_mannheim.korap.util.QueryException;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.tree.ParseTree;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.reflect.Method;
import java.util.*;

/**
 * Map representation of Poliqarp syntax tree as returned by ANTLR
 *
 * @author joachim
 */
public class PoliqarpPlusTree extends Antlr4AbstractSyntaxTree {

    Logger log = LoggerFactory.getLogger(PoliqarpPlusTree.class);
    /**
     * Top-level map representing the whole request.
     */
    LinkedHashMap<String, Object> requestMap = new LinkedHashMap<String, Object>();
    /**
     * Keeps track of open node categories
     */
    LinkedList<String> openNodeCats = new LinkedList<String>();
    /**
     * Flag that indicates whether token fields or meta fields are currently being processed
     */
    boolean inMeta = false;
    /**
     * Flag that indicates whether a cq_segment is to be ignored (e.g. when it is empty, is followed directly by only a spanclass and has no other children etc...).
     */
    boolean ignoreCq_segment = false;
    /**
     * Flag that indicates whether a cq_segments element is quantified by an occ element.
     */
    boolean cqHasOccSibling = false;
    /**
     * Flag that indicates whether a cq_segments' children are quantified by an occ element.
     */
    boolean cqHasOccChild = false;
    /**
     * Flag for negation of complete field
     */
    boolean negField = false;
    /**
     * Flag that indicates whether subsequent element is to be aligned.
     */
    boolean alignNext = false;
    /**
     * Flag that indicates whether current element has been aligned.
     */
    boolean isAligned = false;
    /**
     * Indicates a sequence which has an align operator as its child. Needed for deciding
     * when to close the align group object.
     */
//	ParseTree alignedSequence = null;
    /**
     * Parser object deriving the ANTLR parse tree.
     */
    Parser parser;
    /**
     * Keeps track of all visited nodes in a tree
     */
    List<ParseTree> visited = new ArrayList<ParseTree>();

    /**
     * Keeps track of active fields (like 'base=foo').
     */
    LinkedList<ArrayList<Object>> fieldStack = new LinkedList<ArrayList<Object>>();
    /**
     * Keeps track of active tokens.
     */
    LinkedList<LinkedHashMap<String, Object>> tokenStack = new LinkedList<LinkedHashMap<String, Object>>();
    /**
     * Marks the currently active token in order to know where to add flags (might already have been taken away from token stack).
     */
    LinkedHashMap<String, Object> curToken = new LinkedHashMap<String, Object>();
    /**
     * Keeps track of active object.
     */
    LinkedList<LinkedHashMap<String, Object>> objectStack = new LinkedList<LinkedHashMap<String, Object>>();
    /**
     * Marks the object to which following occurrence information is to be added.
     */
    LinkedHashMap<String, Object> curOccGroup = new LinkedHashMap<String, Object>();
    /**
     * Keeps track of how many objects there are to pop after every recursion of {@link #processNode(ParseTree)}
     */
    LinkedList<Integer> objectsToPop = new LinkedList<Integer>();
    /**
     * Keeps track of how many objects there are to pop after every recursion of {@link #processNode(ParseTree)}
     */
    LinkedList<Integer> tokensToPop = new LinkedList<Integer>();
    /**
     * Keeps track of how many objects there are to pop after every recursion of {@link #processNode(ParseTree)}
     */
    LinkedList<Integer> fieldsToPop = new LinkedList<Integer>();
    /**
     * If true, print debug statements
     */
    public static boolean verbose = false;
    /**
     * Index of the current child of its parent (needed for relating occ elements to their operands).
     */
    int curChildIndex = 0;
    /**
     *
     */
    Integer stackedObjects = 0;
    Integer stackedTokens = 0;
    Integer stackedFields = 0;


    /**
     * Most centrally, this class maintains a set of nested maps and lists which represent the JSON tree, which is built by the JSON serialiser
     * on basis of the {@link #requestMap} at the root of the tree.
     * <br/>
     * The class further maintains a set of stacks which effectively keep track of which objects to embed in which containing objects.
     *
     * @param query The syntax tree as returned by ANTLR
     * @throws QueryException
     */
    public PoliqarpPlusTree(String query) throws QueryException {
        try {
            process(query);
        } catch (NullPointerException e) {
            if (query.contains(" ")) {
                System.err.println("Warning: It seems like your query contains illegal whitespace characters. Trying again with whitespaces removed...");
                query = query.replaceAll(" ", "");
                process(query);
            } else {
                throw new QueryException("Error handling query.");
            }
        }
        System.out.println(">>> " + requestMap.get("query") + " <<<");
        log.info(">>> " + requestMap.get("query") + " <<<");
    }

    @Override
    public Map<String, Object> getRequestMap() {
        return requestMap;
    }

    @Override
    public void process(String query) throws QueryException {
        ParseTree tree;
        try {
            tree = parsePoliqarpQuery(query);
        } catch (QueryException e) {
            // if the second time query could not be parsed, throw exception!
            tree = parsePoliqarpQuery(query.replaceAll(" ", ""));
        }
        super.parser = this.parser;
        log.info("Processing PoliqarpPlus");
        System.out.println("Processing PoliqarpPlus");
        requestMap.put("@context", "http://ids-mannheim.de/ns/KorAP/json-ld/v0.1/context.jsonld");
//		prepareContext(requestMap);
        processNode(tree);
    }

    /**
     * Recursively calls itself with the children of the currently active node, traversing the tree nodes in a top-down, depth-first fashion.
     * A list is maintained that contains all visited nodes
     * in case they have been directly addressed by its (grand-/grand-grand-/...) parent node, such that some processing time is saved, as these node will
     * not be processed. This method is effectively a list of if-statements that are responsible for treating the different node types correctly and filling the
     * respective maps/lists.
     *
     * @param node The currently processed node. The process(String query) method calls this method with the root.
     * @throws QueryException
     */
    @SuppressWarnings("unchecked")
    private void processNode(ParseTree node) throws QueryException {
        // Top-down processing
        if (visited.contains(node)) return;
        else visited.add(node);

        if (alignNext) {
            alignNext = false;
            isAligned = true;
        }

        String nodeCat = getNodeCat(node);
        openNodeCats.push(nodeCat);

        stackedObjects = 0;
        stackedTokens = 0;
        stackedFields = 0;

        if (verbose) {
            System.err.println(" " + objectStack);
//			System.err.println(" "+tokenStack);
            System.out.println(openNodeCats);
        }


		/*
         ****************************************************************
		 **************************************************************** 
		 * 			Processing individual node categories  				*
		 ****************************************************************
		 ****************************************************************
		 */

        // cq_segments/sq_segments: token group
        if (nodeCat.equals("cq_segments") || nodeCat.equals("sq_segments")) {
            cqHasOccSibling = false;
            cqHasOccChild = false;
            // disregard empty segments in simple queries (parsed by ANTLR as empty cq_segments)
            ignoreCq_segment = (node.getChildCount() == 1 && (node.getChild(0).toStringTree(parser).equals(" ") || getNodeCat(node.getChild(0)).equals("spanclass") || getNodeCat(node.getChild(0)).equals("position")));
//            ignoreCq_segment = (node.getChildCount() == 1 && (node.getChild(0).toStringTree(parser).equals(" ") || getNodeCat(node.getChild(0)).equals("position")));
            // ignore this node if it only serves as an aligned sequence container
            if (node.getChildCount() > 1) {
                if (getNodeCat(node.getChild(1)).equals("cq_segments") && hasChild(node.getChild(1), "alignment")) {
                    ignoreCq_segment = true;
                }
                if (getNodeCat(node.getChild(0)).equals("(") && getNodeCat(node.getChild(node.getChildCount()-1)).equals(")")) {
                    ignoreCq_segment = true;
                }
            }
            System.err.println("WAAAAAAH"+objectStack);
            if (!ignoreCq_segment) {
                LinkedHashMap<String, Object> sequence = new LinkedHashMap<String, Object>();
                // Step 0:  cq_segments has 'occ' child -> introduce group as super group to the sequence/token/group
                // this requires creating a group and inserting it at a suitable place
                if (node.getParent().getChildCount() > curChildIndex + 2 && getNodeCat(node.getParent().getChild(curChildIndex + 2)).equals("occ")) {
                    cqHasOccSibling = true;
                    createOccGroup(node);
                }
                if (getNodeCat(node.getChild(node.getChildCount() - 1)).equals("occ")) {
                    cqHasOccChild = true;
                }
                // Step I: decide type of element (one or more elements? -> token or sequence)
                // take into account a possible 'occ' child with accompanying parantheses, therefore 3 extra children
                int occExtraChildren = cqHasOccChild ? 3 : 0;
                if (node.getChildCount() > 1 + occExtraChildren) {
                    ParseTree emptySegments = getFirstChildWithCat(node, "empty_segments");
                    if (emptySegments != null && emptySegments != node.getChild(0)) {
                        String[] minmax = parseEmptySegments(emptySegments);
                        Integer min = Integer.parseInt(minmax[0]);
                        Integer max = Integer.parseInt(minmax[1]);
                        sequence.put("@type", "korap:group");
                        sequence.put("operation", "operation:sequence");
                        sequence.put("inOrder", true);
                        ArrayList<Object> constraint = new ArrayList<Object>();
                        sequence.put("distances", constraint);
                        ArrayList<Object> sequenceOperands = new ArrayList<Object>();
                        sequence.put("operands", sequenceOperands);
                        objectStack.push(sequence);
                        stackedObjects++;
                        LinkedHashMap<String, Object> distMap = new LinkedHashMap<String, Object>();
                        constraint.add(distMap);
                        distMap.put("@type", "korap:distance");
                        distMap.put("key", "w");
                        distMap.put("min", min);
                        distMap.put("max", max);
                    } else {
                        sequence.put("@type", "korap:group");
                        sequence.put("operation", "operation:" + "sequence");
                        ArrayList<Object> sequenceOperands = new ArrayList<Object>();
                        if (emptySegments != null) {
                            String[] minmax = parseEmptySegments(emptySegments);
                            Integer min = Integer.parseInt(minmax[0]);
                            Integer max = Integer.parseInt(minmax[1]);
                            sequence.put("offset-min", min - 1);
                            sequence.put("offset-max", max - 1);
                        }
                        sequence.put("operands", sequenceOperands);
                        objectStack.push(sequence);
                        stackedObjects++;
                    }
                } else {
                    // if only child, make the sequence a mere token...
                    // ... but only if it has a real token/element beneath it
                    if (!isContainerOnly(node)) {
                        sequence.put("@type", "korap:token");
                        tokenStack.push(sequence);
                        stackedTokens++;
                        objectStack.push(sequence);
                        stackedObjects++;
                        // else, it's a group (with shrink()/spanclass/align... as child)
                    } else {
//						sequence.put("@type", "korap:group");
//						objectStack.push(sequence);
//						stackedObjects++;
                    }
                }
                // Step II: decide where to put this element
                // check if this is an argument for a containing occurrence group (see step 0)
                if (cqHasOccSibling) {
                    ArrayList<Object> topGroupOperands = (ArrayList<Object>) objectStack.get(1).get("operands");
                    topGroupOperands.add(sequence);
                    // ...if not modified by occurrence, put into appropriate super object
                } else {
                    if (openNodeCats.get(1).equals("query")) {
                        // cq_segment is top query node
                        if (node.getParent().getChildCount() == 1) {
                            // only child
                            requestMap.put("query", sequence);
                        } else {
                            // not an only child, need to create containing sequence
                            if (node.getParent().getChild(0).equals(node)) {
                                // if first child, create containing sequence and embed there
                                LinkedHashMap<String, Object> superSequence = new LinkedHashMap<String, Object>();
                                superSequence.put("@type", "korap:group");
                                superSequence.put("operation", "operation:" + "sequence");
                                ArrayList<Object> operands = new ArrayList<Object>();
                                superSequence.put("operands", operands);
                                operands.add(sequence);
                                requestMap.put("query", superSequence);
                                objectStack.push(superSequence); // add at 2nd position to keep current cq_segment accessible
                                stackedObjects++;
                            } else {
                                // if not first child, add to previously created parent sequence
                                ArrayList<Object> topSequenceOperands;
                                try {
                                    topSequenceOperands = (ArrayList<Object>) objectStack.get(1).get("operands");
                                } catch (IndexOutOfBoundsException e) {
                                    // Normally, the current element has been added to the object stack, so the try-block works fine.
                                    // In some cases however, the element is not added (see ultimate else-block in Step I), and we need a
                                    // fallback to the first element in the object stack.
                                    topSequenceOperands = (ArrayList<Object>) objectStack.get(0).get("operands");
                                }

                                topSequenceOperands.add(sequence);
                            }
                        }
                    } else if (!objectStack.isEmpty()) {
                        // embed in super sequence
                    	System.err.println("BBBBUUUHH "+isContainerOnly(node));

                        ArrayList<Object> topSequenceOperands;
                        if (!isContainerOnly(node)) {
                            try {
                                topSequenceOperands = (ArrayList<Object>) objectStack.get(1).get("operands");
                                topSequenceOperands.add(sequence);
                            } catch (IndexOutOfBoundsException e) {
//								topSequenceOperands = (ArrayList<Object>) objectStack.get(0).get("operands");
                            }
                        }


                    }
                }
            }
        }

        // cq_segment
        if (nodeCat.equals("cq_segment")) {
            int onTopOfObjectStack = 0;
            // Step I: determine whether to create new token or get token from the stack (if added by cq_segments)
            LinkedHashMap<String, Object> token;
            if (tokenStack.isEmpty()) {
                token = new LinkedHashMap<String, Object>();
                tokenStack.push(token);
                stackedTokens++;<commit_msg>removed comments<commit_after>package de.ids_mannheim.korap.query.serialize;

import de.ids_mannheim.korap.query.PoliqarpPlusLexer;
import de.ids_mannheim.korap.query.PoliqarpPlusParser;
import de.ids_mannheim.korap.util.QueryException;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.tree.ParseTree;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.reflect.Method;
import java.util.*;

/**
 * Map representation of Poliqarp syntax tree as returned by ANTLR
 *
 * @author joachim
 */
public class PoliqarpPlusTree extends Antlr4AbstractSyntaxTree {

    Logger log = LoggerFactory.getLogger(PoliqarpPlusTree.class);
    /**
     * Top-level map representing the whole request.
     */
    LinkedHashMap<String, Object> requestMap = new LinkedHashMap<String, Object>();
    /**
     * Keeps track of open node categories
     */
    LinkedList<String> openNodeCats = new LinkedList<String>();
    /**
     * Flag that indicates whether token fields or meta fields are currently being processed
     */
    boolean inMeta = false;
    /**
     * Flag that indicates whether a cq_segment is to be ignored (e.g. when it is empty, is followed directly by only a spanclass and has no other children etc...).
     */
    boolean ignoreCq_segment = false;
    /**
     * Flag that indicates whether a cq_segments element is quantified by an occ element.
     */
    boolean cqHasOccSibling = false;
    /**
     * Flag that indicates whether a cq_segments' children are quantified by an occ element.
     */
    boolean cqHasOccChild = false;
    /**
     * Flag for negation of complete field
     */
    boolean negField = false;
    /**
     * Flag that indicates whether subsequent element is to be aligned.
     */
    boolean alignNext = false;
    /**
     * Flag that indicates whether current element has been aligned.
     */
    boolean isAligned = false;
    /**
     * Indicates a sequence which has an align operator as its child. Needed for deciding
     * when to close the align group object.
     */
//	ParseTree alignedSequence = null;
    /**
     * Parser object deriving the ANTLR parse tree.
     */
    Parser parser;
    /**
     * Keeps track of all visited nodes in a tree
     */
    List<ParseTree> visited = new ArrayList<ParseTree>();

    /**
     * Keeps track of active fields (like 'base=foo').
     */
    LinkedList<ArrayList<Object>> fieldStack = new LinkedList<ArrayList<Object>>();
    /**
     * Keeps track of active tokens.
     */
    LinkedList<LinkedHashMap<String, Object>> tokenStack = new LinkedList<LinkedHashMap<String, Object>>();
    /**
     * Marks the currently active token in order to know where to add flags (might already have been taken away from token stack).
     */
    LinkedHashMap<String, Object> curToken = new LinkedHashMap<String, Object>();
    /**
     * Keeps track of active object.
     */
    LinkedList<LinkedHashMap<String, Object>> objectStack = new LinkedList<LinkedHashMap<String, Object>>();
    /**
     * Marks the object to which following occurrence information is to be added.
     */
    LinkedHashMap<String, Object> curOccGroup = new LinkedHashMap<String, Object>();
    /**
     * Keeps track of how many objects there are to pop after every recursion of {@link #processNode(ParseTree)}
     */
    LinkedList<Integer> objectsToPop = new LinkedList<Integer>();
    /**
     * Keeps track of how many objects there are to pop after every recursion of {@link #processNode(ParseTree)}
     */
    LinkedList<Integer> tokensToPop = new LinkedList<Integer>();
    /**
     * Keeps track of how many objects there are to pop after every recursion of {@link #processNode(ParseTree)}
     */
    LinkedList<Integer> fieldsToPop = new LinkedList<Integer>();
    /**
     * If true, print debug statements
     */
    public static boolean verbose = false;
    /**
     * Index of the current child of its parent (needed for relating occ elements to their operands).
     */
    int curChildIndex = 0;
    /**
     *
     */
    Integer stackedObjects = 0;
    Integer stackedTokens = 0;
    Integer stackedFields = 0;


    /**
     * Most centrally, this class maintains a set of nested maps and lists which represent the JSON tree, which is built by the JSON serialiser
     * on basis of the {@link #requestMap} at the root of the tree.
     * <br/>
     * The class further maintains a set of stacks which effectively keep track of which objects to embed in which containing objects.
     *
     * @param query The syntax tree as returned by ANTLR
     * @throws QueryException
     */
    public PoliqarpPlusTree(String query) throws QueryException {
        try {
            process(query);
        } catch (NullPointerException e) {
            if (query.contains(" ")) {
                System.err.println("Warning: It seems like your query contains illegal whitespace characters. Trying again with whitespaces removed...");
                query = query.replaceAll(" ", "");
                process(query);
            } else {
                throw new QueryException("Error handling query.");
            }
        }
        System.out.println(">>> " + requestMap.get("query") + " <<<");
        log.info(">>> " + requestMap.get("query") + " <<<");
    }

    @Override
    public Map<String, Object> getRequestMap() {
        return requestMap;
    }

    @Override
    public void process(String query) throws QueryException {
        ParseTree tree;
        try {
            tree = parsePoliqarpQuery(query);
        } catch (QueryException e) {
            // if the second time query could not be parsed, throw exception!
            tree = parsePoliqarpQuery(query.replaceAll(" ", ""));
        }
        super.parser = this.parser;
        log.info("Processing PoliqarpPlus");
        System.out.println("Processing PoliqarpPlus");
        requestMap.put("@context", "http://ids-mannheim.de/ns/KorAP/json-ld/v0.1/context.jsonld");
//		prepareContext(requestMap);
        processNode(tree);
    }

    /**
     * Recursively calls itself with the children of the currently active node, traversing the tree nodes in a top-down, depth-first fashion.
     * A list is maintained that contains all visited nodes
     * in case they have been directly addressed by its (grand-/grand-grand-/...) parent node, such that some processing time is saved, as these node will
     * not be processed. This method is effectively a list of if-statements that are responsible for treating the different node types correctly and filling the
     * respective maps/lists.
     *
     * @param node The currently processed node. The process(String query) method calls this method with the root.
     * @throws QueryException
     */
    @SuppressWarnings("unchecked")
    private void processNode(ParseTree node) throws QueryException {
        // Top-down processing
        if (visited.contains(node)) return;
        else visited.add(node);

        if (alignNext) {
            alignNext = false;
            isAligned = true;
        }

        String nodeCat = getNodeCat(node);
        openNodeCats.push(nodeCat);

        stackedObjects = 0;
        stackedTokens = 0;
        stackedFields = 0;

        if (verbose) {
            System.err.println(" " + objectStack);
//			System.err.println(" "+tokenStack);
            System.out.println(openNodeCats);
        }


		/*
         ****************************************************************
		 **************************************************************** 
		 * 			Processing individual node categories  				*
		 ****************************************************************
		 ****************************************************************
		 */

        // cq_segments/sq_segments: token group
        if (nodeCat.equals("cq_segments") || nodeCat.equals("sq_segments")) {
            cqHasOccSibling = false;
            cqHasOccChild = false;
            // disregard empty segments in simple queries (parsed by ANTLR as empty cq_segments)
            ignoreCq_segment = (node.getChildCount() == 1 && (node.getChild(0).toStringTree(parser).equals(" ") || getNodeCat(node.getChild(0)).equals("spanclass") || getNodeCat(node.getChild(0)).equals("position")));
//            ignoreCq_segment = (node.getChildCount() == 1 && (node.getChild(0).toStringTree(parser).equals(" ") || getNodeCat(node.getChild(0)).equals("position")));
            // ignore this node if it only serves as an aligned sequence container
            if (node.getChildCount() > 1) {
                if (getNodeCat(node.getChild(1)).equals("cq_segments") && hasChild(node.getChild(1), "alignment")) {
                    ignoreCq_segment = true;
                }
                if (getNodeCat(node.getChild(0)).equals("(") && getNodeCat(node.getChild(node.getChildCount()-1)).equals(")")) {
                    ignoreCq_segment = true;
                }
            }
            if (!ignoreCq_segment) {
                LinkedHashMap<String, Object> sequence = new LinkedHashMap<String, Object>();
                // Step 0:  cq_segments has 'occ' child -> introduce group as super group to the sequence/token/group
                // this requires creating a group and inserting it at a suitable place
                if (node.getParent().getChildCount() > curChildIndex + 2 && getNodeCat(node.getParent().getChild(curChildIndex + 2)).equals("occ")) {
                    cqHasOccSibling = true;
                    createOccGroup(node);
                }
                if (getNodeCat(node.getChild(node.getChildCount() - 1)).equals("occ")) {
                    cqHasOccChild = true;
                }
                // Step I: decide type of element (one or more elements? -> token or sequence)
                // take into account a possible 'occ' child with accompanying parantheses, therefore 3 extra children
                int occExtraChildren = cqHasOccChild ? 3 : 0;
                if (node.getChildCount() > 1 + occExtraChildren) {
                    ParseTree emptySegments = getFirstChildWithCat(node, "empty_segments");
                    if (emptySegments != null && emptySegments != node.getChild(0)) {
                        String[] minmax = parseEmptySegments(emptySegments);
                        Integer min = Integer.parseInt(minmax[0]);
                        Integer max = Integer.parseInt(minmax[1]);
                        sequence.put("@type", "korap:group");
                        sequence.put("operation", "operation:sequence");
                        sequence.put("inOrder", true);
                        ArrayList<Object> constraint = new ArrayList<Object>();
                        sequence.put("distances", constraint);
                        ArrayList<Object> sequenceOperands = new ArrayList<Object>();
                        sequence.put("operands", sequenceOperands);
                        objectStack.push(sequence);
                        stackedObjects++;
                        LinkedHashMap<String, Object> distMap = new LinkedHashMap<String, Object>();
                        constraint.add(distMap);
                        distMap.put("@type", "korap:distance");
                        distMap.put("key", "w");
                        distMap.put("min", min);
                        distMap.put("max", max);
                    } else {
                        sequence.put("@type", "korap:group");
                        sequence.put("operation", "operation:" + "sequence");
                        ArrayList<Object> sequenceOperands = new ArrayList<Object>();
                        if (emptySegments != null) {
                            String[] minmax = parseEmptySegments(emptySegments);
                            Integer min = Integer.parseInt(minmax[0]);
                            Integer max = Integer.parseInt(minmax[1]);
                            sequence.put("offset-min", min - 1);
                            sequence.put("offset-max", max - 1);
                        }
                        sequence.put("operands", sequenceOperands);
                        objectStack.push(sequence);
                        stackedObjects++;
                    }
                } else {
                    // if only child, make the sequence a mere token...
                    // ... but only if it has a real token/element beneath it
                    if (!isContainerOnly(node)) {
                        sequence.put("@type", "korap:token");
                        tokenStack.push(sequence);
                        stackedTokens++;
                        objectStack.push(sequence);
                        stackedObjects++;
                        // else, it's a group (with shrink()/spanclass/align... as child)
                    } else {
//						sequence.put("@type", "korap:group");
//						objectStack.push(sequence);
//						stackedObjects++;
                    }
                }
                // Step II: decide where to put this element
                // check if this is an argument for a containing occurrence group (see step 0)
                if (cqHasOccSibling) {
                    ArrayList<Object> topGroupOperands = (ArrayList<Object>) objectStack.get(1).get("operands");
                    topGroupOperands.add(sequence);
                    // ...if not modified by occurrence, put into appropriate super object
                } else {
                    if (openNodeCats.get(1).equals("query")) {
                        // cq_segment is top query node
                        if (node.getParent().getChildCount() == 1) {
                            // only child
                            requestMap.put("query", sequence);
                        } else {
                            // not an only child, need to create containing sequence
                            if (node.getParent().getChild(0).equals(node)) {
                                // if first child, create containing sequence and embed there
                                LinkedHashMap<String, Object> superSequence = new LinkedHashMap<String, Object>();
                                superSequence.put("@type", "korap:group");
                                superSequence.put("operation", "operation:" + "sequence");
                                ArrayList<Object> operands = new ArrayList<Object>();
                                superSequence.put("operands", operands);
                                operands.add(sequence);
                                requestMap.put("query", superSequence);
                                objectStack.push(superSequence); // add at 2nd position to keep current cq_segment accessible
                                stackedObjects++;
                            } else {
                                // if not first child, add to previously created parent sequence
                                ArrayList<Object> topSequenceOperands;
                                try {
                                    topSequenceOperands = (ArrayList<Object>) objectStack.get(1).get("operands");
                                } catch (IndexOutOfBoundsException e) {
                                    // Normally, the current element has been added to the object stack, so the try-block works fine.
                                    // In some cases however, the element is not added (see ultimate else-block in Step I), and we need a
                                    // fallback to the first element in the object stack.
                                    topSequenceOperands = (ArrayList<Object>) objectStack.get(0).get("operands");
                                }

                                topSequenceOperands.add(sequence);
                            }
                        }
                    } else if (!objectStack.isEmpty()) {
                        // embed in super sequence
                        ArrayList<Object> topSequenceOperands;
                        if (!isContainerOnly(node)) {
                            try {
                                topSequenceOperands = (ArrayList<Object>) objectStack.get(1).get("operands");
                                topSequenceOperands.add(sequence);
                            } catch (IndexOutOfBoundsException e) {
//								topSequenceOperands = (ArrayList<Object>) objectStack.get(0).get("operands");
                            }
                        }


                    }
                }
            }
        }

        // cq_segment
        if (nodeCat.equals("cq_segment")) {
            int onTopOfObjectStack = 0;
            // Step I: determine whether to create new token or get token from the stack (if added by cq_segments)
            LinkedHashMap<String, Object> token;
            if (tokenStack.isEmpty()) {
                token = new LinkedHashMap<String, Object>();
                tokenStack.push(token);
                stackedTokens++;
{'fraction_non_alphanumeric': 0.1135663350647064, 'fraction_numerical': 0.00305191184670012, 'mean_word_length': 1.8017758776617612, 'pattern_counts': {'":': 0, '<': 123, '<?xml version=': 0, '>': 133, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2017403', 'n_tokens_mistral': 8438, 'n_tokens_neox': 8083, 'n_words': 2381}