<commit_before>/**
 *  Copyright (C) 2004 - 2005 Orbeon, Inc.
 *
 *  This program is free software; you can redistribute it and/or modify it under the terms of the
 *  GNU Lesser General Public License as published by the Free Software Foundation; either version
 *  2.1 of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Lesser General Public License for more details.
 *
 *  The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
 */
package org.orbeon.oxf.processor.pipeline;

import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.Node;
import org.orbeon.oxf.cache.*;
import org.orbeon.oxf.common.OXFException;
import org.orbeon.oxf.common.ValidationException;
import org.orbeon.oxf.pipeline.api.PipelineContext;
import org.orbeon.oxf.processor.*;
import org.orbeon.oxf.processor.generator.DOMGenerator;
import org.orbeon.oxf.processor.pipeline.ast.*;
import org.orbeon.oxf.util.LoggerFactory;
import org.orbeon.oxf.util.XPathCache;
import org.orbeon.oxf.util.PooledXPathExpression;
import org.orbeon.oxf.xml.ForwardingContentHandler;
import org.orbeon.oxf.xml.dom4j.Dom4jUtils;
import org.orbeon.saxon.dom4j.DocumentWrapper;
import org.orbeon.saxon.xpath.XPathException;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import java.util.*;

public class ForEachProcessor extends ProcessorImpl implements AbstractProcessor {

    static private Logger logger = LoggerFactory.createLogger(ForEachProcessor.class);
    public static final String CURRENT = "$current";
    public static final String FOR_EACH_DATA_INPUT = "$data";

    private ASTForEach forEachAST;
    private Object validity;

    public ForEachProcessor(ASTForEach forEachAST, Object validity) {
        this.forEachAST = forEachAST;
        this.validity = validity;
        setLocationData(forEachAST.getLocationData());
    }

    public Processor createInstance(PipelineContext context) {
        return new ConcreteForEachProcessor(forEachAST, validity);
    }

    public static class ConcreteForEachProcessor extends ProcessorImpl {

        Processor iterationProcessor;
        private ProcessorOutput iterationOutput;
        private String select;
        private Map namespaceContext;
        private String rootLocalName;
        private String rootQName;
        private String rootNamespaceURI;

        public ConcreteForEachProcessor(ASTForEach forEachAST, Object validity) {
            String[] refsWithNoId = getRefsWithNoId(forEachAST);
            String idOrRef = forEachAST.getId() != null ? forEachAST.getId() : forEachAST.getRef();

            // Create pipeline for content of for-each (the "iteration processor")
            {
                ASTPipeline astPipeline = new ASTPipeline();
                astPipeline.setValidity(validity);
                astPipeline.getStatements().addAll(forEachAST.getStatements());
                astPipeline.setNode(forEachAST.getNode());
                for (int i = 0; i < refsWithNoId.length; i++) {
                    astPipeline.addParam(new ASTParam(ASTParam.INPUT, refsWithNoId[i]));
                    if (! refsWithNoId[i].equals(CURRENT))
                        addInputInfo(new ProcessorInputOutputInfo(refsWithNoId[i]));
                }
                if (idOrRef != null) {
                    astPipeline.addParam(new ASTParam(ASTParam.OUTPUT, idOrRef));
                    addOutputInfo(new ProcessorInputOutputInfo(idOrRef));
                }
                if (logger.isDebugEnabled()) {
                    ASTDocumentHandler astDocumentHandler = new ASTDocumentHandler();
                    astPipeline.walk(astDocumentHandler);
                    logger.debug("Iteration pipeline:\n" + Dom4jUtils.domToString(astDocumentHandler.getDocument()));
                }
                iterationProcessor = new PipelineProcessor(astPipeline);
            }

            // Connect processor inputs to inputs of this processor
            for (int i = 0; i < refsWithNoId.length; i++) {
                if (! refsWithNoId[i].equals(CURRENT)) {
                    ProcessorInput pipelineInput = iterationProcessor.createInput(refsWithNoId[i]);
                    pipelineInput.setOutput(new ForwardingProcessorOutput(refsWithNoId[i]));
                }
            }

            // Connect processor to the IterationProcessorOutput
            ProcessorInput iterationInput = iterationProcessor.createInput(CURRENT);
            ProcessorOutput currentOutput = new IterationProcessorOutput(ForEachProcessor.class, CURRENT);
            currentOutput.setInput(iterationInput);
            iterationInput.setOutput(currentOutput);
            iterationOutput = iterationProcessor.createOutput(idOrRef);

            select = forEachAST.getSelect();
            namespaceContext = Dom4jUtils.getNamespaceContext((Element) forEachAST.getNode());
            if (forEachAST.getRoot() != null) {
                rootQName = forEachAST.getRoot();
                int columnPosition = rootQName.indexOf(':');
                if (columnPosition == -1) {
                    // No prefix
                    rootLocalName = rootQName;
                    rootNamespaceURI = "";
                } else {
                    // Extract prefix, find namespace URI
                    String prefix = rootQName.substring(0, columnPosition);
                    rootNamespaceURI = (String) namespaceContext.get(prefix);
                    if (rootNamespaceURI == null)
                        throw new ValidationException("Prefix '" + prefix + "' used in root attribute is undefined", forEachAST.getLocationData());
                    rootLocalName = rootQName.substring(columnPosition + 1);
                }
            }
        }

        public ProcessorOutput createOutput( final String name ) {
            ProcessorOutput output = new ProcessorImpl.ProcessorOutputImpl(getClass(), name) {
                public void readImpl(PipelineContext context, ContentHandler contentHandler) {
                    try {
                        State state = (State) getState(context);
                        initDOMGenerators(context, state);

                        // Read n times from iterationOutput
                        contentHandler.startDocument();
                        contentHandler.startElement(rootNamespaceURI, rootLocalName, rootQName, new AttributesImpl());
                        for (int i = 0; i < state.domGenerators.size(); i++) {
                            state.currentDOMGenerator = i;
                            iterationProcessor.reset(context);
                            iterationOutput.read(context, new ForwardingContentHandler(contentHandler) {
                                public void startDocument() {}
                                public void endDocument() {}
                            });
                        }
                        contentHandler.endElement(rootNamespaceURI, rootLocalName, rootQName);
                        contentHandler.endDocument();
                    } catch (SAXException e) {
                        throw new OXFException(e);
                    }
                }

                /**
                 * Combine keys from all iterations
                 */
                protected OutputCacheKey getKeyImpl(PipelineContext context) {
                    State state = (State) getState(context);
                    updateStateWithDOMGeneratorsFromCache
                            (context, state, createInternalKey(context), createInternalValidity(context));
                    if (state.domGenerators == null)  return null;
                    List keys = new ArrayList();
                    for (int i = 0; i < state.domGenerators.size(); i++) {
                        state.currentDOMGenerator = i;
                        iterationProcessor.reset(context);
                        OutputCacheKey key = ((Cacheable) iterationOutput).getKey(context);
                        if (key == null) return null;
                        keys.add(key);
                    }
                    final CacheKey[] outKys = new CacheKey[ keys.size() ];
                    keys.toArray( outKys );
                    return new CompoundOutputCacheKey( ForEachProcessor.class, name, outKys );
                }

                /**
                 * Combine validities from all iterations
                 */
                protected Object getValidityImpl(PipelineContext context) {
                    State state = (State) getState(context);
                    updateStateWithDOMGeneratorsFromCache
                            (context, state, createInternalKey(context), createInternalValidity(context));
                    if (state.domGenerators == null)  return null;
                    List validities = new ArrayList();
                    for (int i = 0; i < state.domGenerators.size(); i++) {
                        state.currentDOMGenerator = i;
                        iterationProcessor.reset(context);
                        Object valdity = ((Cacheable) iterationOutput).getValidity(context);
                        if (valdity == null) return null;
                        validities.add(valdity);
                    }
                    return validities;
                }
<commit_msg>Make for-each processor non-cachable<commit_after>/**
 *  Copyright (C) 2004 - 2005 Orbeon, Inc.
 *
 *  This program is free software; you can redistribute it and/or modify it under the terms of the
 *  GNU Lesser General Public License as published by the Free Software Foundation; either version
 *  2.1 of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Lesser General Public License for more details.
 *
 *  The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
 */
package org.orbeon.oxf.processor.pipeline;

import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.Node;
import org.orbeon.oxf.cache.*;
import org.orbeon.oxf.common.OXFException;
import org.orbeon.oxf.common.ValidationException;
import org.orbeon.oxf.pipeline.api.PipelineContext;
import org.orbeon.oxf.processor.*;
import org.orbeon.oxf.processor.generator.DOMGenerator;
import org.orbeon.oxf.processor.pipeline.ast.*;
import org.orbeon.oxf.util.LoggerFactory;
import org.orbeon.oxf.util.XPathCache;
import org.orbeon.oxf.util.PooledXPathExpression;
import org.orbeon.oxf.xml.ForwardingContentHandler;
import org.orbeon.oxf.xml.dom4j.Dom4jUtils;
import org.orbeon.saxon.dom4j.DocumentWrapper;
import org.orbeon.saxon.xpath.XPathException;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import java.util.*;

public class ForEachProcessor extends ProcessorImpl implements AbstractProcessor {

    static private Logger logger = LoggerFactory.createLogger(ForEachProcessor.class);
    public static final String CURRENT = "$current";
    public static final String FOR_EACH_DATA_INPUT = "$data";

    private ASTForEach forEachAST;
    private Object validity;

    public ForEachProcessor(ASTForEach forEachAST, Object validity) {
        this.forEachAST = forEachAST;
        this.validity = validity;
        setLocationData(forEachAST.getLocationData());
    }

    public Processor createInstance(PipelineContext context) {
        return new ConcreteForEachProcessor(forEachAST, validity);
    }

    public static class ConcreteForEachProcessor extends ProcessorImpl {

        Processor iterationProcessor;
        private ProcessorOutput iterationOutput;
        private String select;
        private Map namespaceContext;
        private String rootLocalName;
        private String rootQName;
        private String rootNamespaceURI;

        public ConcreteForEachProcessor(ASTForEach forEachAST, Object validity) {
            String[] refsWithNoId = getRefsWithNoId(forEachAST);
            String idOrRef = forEachAST.getId() != null ? forEachAST.getId() : forEachAST.getRef();

            // Create pipeline for content of for-each (the "iteration processor")
            {
                ASTPipeline astPipeline = new ASTPipeline();
                astPipeline.setValidity(validity);
                astPipeline.getStatements().addAll(forEachAST.getStatements());
                astPipeline.setNode(forEachAST.getNode());
                for (int i = 0; i < refsWithNoId.length; i++) {
                    astPipeline.addParam(new ASTParam(ASTParam.INPUT, refsWithNoId[i]));
                    if (! refsWithNoId[i].equals(CURRENT))
                        addInputInfo(new ProcessorInputOutputInfo(refsWithNoId[i]));
                }
                if (idOrRef != null) {
                    astPipeline.addParam(new ASTParam(ASTParam.OUTPUT, idOrRef));
                    addOutputInfo(new ProcessorInputOutputInfo(idOrRef));
                }
                if (logger.isDebugEnabled()) {
                    ASTDocumentHandler astDocumentHandler = new ASTDocumentHandler();
                    astPipeline.walk(astDocumentHandler);
                    logger.debug("Iteration pipeline:\n" + Dom4jUtils.domToString(astDocumentHandler.getDocument()));
                }
                iterationProcessor = new PipelineProcessor(astPipeline);
            }

            // Connect processor inputs to inputs of this processor
            for (int i = 0; i < refsWithNoId.length; i++) {
                if (! refsWithNoId[i].equals(CURRENT)) {
                    ProcessorInput pipelineInput = iterationProcessor.createInput(refsWithNoId[i]);
                    pipelineInput.setOutput(new ForwardingProcessorOutput(refsWithNoId[i]));
                }
            }

            // Connect processor to the IterationProcessorOutput
            ProcessorInput iterationInput = iterationProcessor.createInput(CURRENT);
            ProcessorOutput currentOutput = new IterationProcessorOutput(ForEachProcessor.class, CURRENT);
            currentOutput.setInput(iterationInput);
            iterationInput.setOutput(currentOutput);
            iterationOutput = iterationProcessor.createOutput(idOrRef);

            select = forEachAST.getSelect();
            namespaceContext = Dom4jUtils.getNamespaceContext((Element) forEachAST.getNode());
            if (forEachAST.getRoot() != null) {
                rootQName = forEachAST.getRoot();
                int columnPosition = rootQName.indexOf(':');
                if (columnPosition == -1) {
                    // No prefix
                    rootLocalName = rootQName;
                    rootNamespaceURI = "";
                } else {
                    // Extract prefix, find namespace URI
                    String prefix = rootQName.substring(0, columnPosition);
                    rootNamespaceURI = (String) namespaceContext.get(prefix);
                    if (rootNamespaceURI == null)
                        throw new ValidationException("Prefix '" + prefix + "' used in root attribute is undefined", forEachAST.getLocationData());
                    rootLocalName = rootQName.substring(columnPosition + 1);
                }
            }
        }

        public ProcessorOutput createOutput( final String name ) {
            ProcessorOutput output = new ProcessorImpl.ProcessorOutputImpl(getClass(), name) {
                public void readImpl(PipelineContext context, ContentHandler contentHandler) {
                    try {
                        State state = (State) getState(context);
                        initDOMGenerators(context, state);

                        // Read n times from iterationOutput
                        contentHandler.startDocument();
                        contentHandler.startElement(rootNamespaceURI, rootLocalName, rootQName, new AttributesImpl());
                        for (int i = 0; i < state.domGenerators.size(); i++) {
                            state.currentDOMGenerator = i;
                            iterationProcessor.reset(context);
                            iterationOutput.read(context, new ForwardingContentHandler(contentHandler) {
                                public void startDocument() {}
                                public void endDocument() {}
                            });
                        }
                        contentHandler.endElement(rootNamespaceURI, rootLocalName, rootQName);
                        contentHandler.endDocument();
                    } catch (SAXException e) {
                        throw new OXFException(e);
                    }
                }

                /**
                 * For each is not cachable. We used to have code here that combines the keys for
                 * the outputs of the different executions of the for-each. The problem is when we
                 * have a block in the for-each that has an output and a serializer. In that case,
                 * getting the output key runs the serializer.
                 *
                 * So if we get the key here, we'll run the serializer n times, which is
                 * unexpected. Maybe an an API that combines reading the output and reading the
                 * key/validity would solve this problem.
                 */
                protected OutputCacheKey getKeyImpl(PipelineContext context) {
                    return null;
                }

                /**
                 * For each is not cachable. See comment in getKeyImpl().
                 */
                protected Object getValidityImpl(PipelineContext context) {
                    return null;
                }

{'fraction_non_alphanumeric': 0.07985881314802559, 'fraction_numerical': 0.002867857930730201, 'mean_word_length': 1.9966947611964965, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6540701', 'n_tokens_mistral': 4461, 'n_tokens_neox': 4241, 'n_words': 1094}