<commit_before>/*
 * This file is part of the DITA Open Toolkit project.
 * See the accompanying license.txt file for applicable licenses.
 */

/*
 * (c) Copyright IBM Corp. 2007 All Rights Reserved.
 */
package org.dita.dost.writer;

import org.dita.dost.exception.DITAOTException;
import org.dita.dost.module.ChunkModule.ChunkFilenameGenerator;
import org.dita.dost.util.Job.FileInfo;
import org.dita.dost.util.TopicIdParser;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.Text;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.AttributesImpl;

import java.io.*;
import java.net.URI;
import java.util.*;

import static javax.xml.XMLConstants.XMLNS_ATTRIBUTE;
import static org.dita.dost.reader.ChunkMapReader.*;
import static org.dita.dost.util.Constants.*;
import static org.dita.dost.util.FileUtils.isAbsolutePath;
import static org.dita.dost.util.URLUtils.*;
import static org.dita.dost.util.XMLUtils.*;
import static org.dita.dost.writer.ImageMetadataFilter.DITA_OT_NS;
import static org.dita.dost.writer.ImageMetadataFilter.DITA_OT_PREFIX;

/**
 * ChunkTopicParser class, writing chunking content into relative topic files
 * and then update list. Not reusable and not thread-safe.
 */
public abstract class AbstractChunkTopicParser extends AbstractXMLWriter {

    /**
     * Keys and values are absolute chimera paths, i.e. systems paths with fragments
     */
    LinkedHashMap<URI, URI> changeTable = null;
    /**
     * Keys and values are absolute chimera paths, i.e. systems paths with fragments
     */
    Map<URI, URI> conflictTable = null;

    Element rootTopicref = null;

    /** Input file's parent absolute path. */
    URI currentFile = null;

    URI currentParsingFile = null;
    /** Absolute temporary output file */
    URI outputFile = null;

    String targetTopicId = null;

    String selectMethod = CHUNK_SELECT_DOCUMENT;
    // flag whether output the nested nodes
    boolean include = false;
    boolean skip = false;

    int includelevel = 0;
    int skipLevel = 0;

    final Set<String> topicSpecSet = new HashSet<>(16);

    boolean startFromFirstTopic = false;

    Writer output = null;

    Set<String> topicID = new HashSet<>();

    Map<String, String> currentParsingFileTopicIDChangeTable;

    private ChunkFilenameGenerator chunkFilenameGenerator;

    abstract public void write(final URI filename) throws DITAOTException;

    @Override
    public void write(final File fileDir) throws DITAOTException {
        throw new UnsupportedOperationException();
    }

    /**
     * Set up the class.
     *
     * @param changeTable   changeTable
     * @param conflictTable conflictTable
     * @param rootTopicref  chunking topicref
     */
    public void setup(final LinkedHashMap<URI, URI> changeTable, final Map<URI, URI> conflictTable,
                      final Element rootTopicref,
                      final ChunkFilenameGenerator chunkFilenameGenerator) {
        this.changeTable = changeTable;
        this.rootTopicref = rootTopicref;
        this.conflictTable = conflictTable;
        this.chunkFilenameGenerator = chunkFilenameGenerator;
    }

    // Filter methods

    @Override
    public void characters(final char[] ch, final int start, final int length) throws SAXException {
        if (include) {
            try {
                output.write(escapeXML(ch, start, length));
            } catch (final IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    @Override
    public void endDocument() throws SAXException {
        include = false;
        skip = false;
    }

    @Override
    public abstract void endElement(final String uri, final String localName, final String qName) throws SAXException;

    @Override
    public void ignorableWhitespace(final char[] ch, final int start, final int length) throws SAXException {
        if (include) {
            try {
                output.write(ch, start, length);
            } catch (final IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    @Override
    public void processingInstruction(final String target, final String data) throws SAXException {
        if (include
                || PI_WORKDIR_TARGET.equals(target)
                || PI_WORKDIR_TARGET_URI.equals(target)
                || PI_PATH2PROJ_TARGET.equals(target)
                || PI_PATH2PROJ_TARGET_URI.equals(target)) {
            try {
                writeProcessingInstruction(output, target, data);
            } catch (final IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    @Override
    public void startDocument() throws SAXException {
        // difference between to-content & select-topic
        if (CHUNK_SELECT_DOCUMENT.equals(selectMethod)) {
            // currentParsingFile can never equal outputFile except when
            // chunk="to-content" is set at map level
            if ((currentParsingFile).equals(outputFile)) {
                // if current file serves as root of new chunk
                // include will be set to true in startDocument()
                // in order to copy PIs and <dita> element
                // otherwise, if current file is copied to other file
                // do not copy PIs and <dita>element
                include = true;
                skip = false;
                skipLevel = 0;
            } else {
                include = false;
                startFromFirstTopic = true;
                skip = false;
                skipLevel = 0;
            }
        }
    }

    @Override
    public abstract void startElement(final String uri, final String localName, final String qName, final Attributes atts)
            throws SAXException;

    void processSelect(String id) {
        if (include) {
            if (CHUNK_SELECT_TOPIC.equals(selectMethod)) {
                // if select method is "select-topic" and current topic is the nested topic in target topic, skip it.
                include = false;
                skipLevel = 1;
                skip = true;
            } else {
                // if select method is "select-document" or "select-branch"
                // and current topic is the nested topic in target topic.
                // if file name has been changed, add an entry in changeTable
                if (!currentParsingFile.equals(outputFile)) {
                    if (id != null) {
                        changeTable.put(setFragment(currentParsingFile, id), setFragment(outputFile, id));
                    } else {
                        changeTable.put(stripFragment(currentParsingFile), stripFragment(outputFile));
                    }
                }
            }
        } else if (skip) {
            skipLevel = 1;
        } else if (id != null && (id.equals(targetTopicId) || startFromFirstTopic)) {
            // if the target topic has not been found and current topic is the target topic
            include = true;
            includelevel = 0;
            skip = false;
            skipLevel = 0;
            startFromFirstTopic = false;
            if (!currentParsingFile.equals(outputFile)) {
                changeTable.put(setFragment(currentParsingFile, id), setFragment(outputFile, id));
            }
        }
    }

    FileInfo generateFileInfo(final URI output) {
        final URI temp = job.tempDirURI.relativize(output);
        final URI result = job.getInputDir().resolve(temp);
        final FileInfo.Builder b = currentParsingFile != null
                ? new FileInfo.Builder(job.getFileInfo(stripFragment(currentParsingFile)))
                : new FileInfo.Builder();
        final FileInfo fi = b
                .uri(temp)
                .result(result)
                .format(ATTR_FORMAT_VALUE_DITA)
                .build();
        return fi;
    }

    URI generateOutputFilename(final String id) {
        URI newFileName = currentFile.resolve(id + FILE_EXTENSION_DITA);
        if (id == null || new File(newFileName).exists()) {
            final URI t = newFileName;
            newFileName = currentFile.resolve(generateFilename());
            conflictTable.put(newFileName, t);
        }
        return newFileName;
    }

    Attributes processAttributes(final Attributes atts) {
        final String cls = atts.getValue(ATTRIBUTE_NAME_CLASS);
        final AttributesImpl resAtts = new AttributesImpl(atts);
        for (int i = 0; i < resAtts.getLength(); i++) {
            final String attrName = resAtts.getQName(i);
            String attrValue = resAtts.getValue(i);

            if (ATTRIBUTE_NAME_ID.equals(attrName)) {
                final String id = attrValue;
                if (TOPIC_TOPIC.matches(cls)) {
                    // change topic @id if there are conflicts.
                    if (topicID.contains(attrValue)) {
                        final String oldAttr = attrValue;
                        attrValue = chunkFilenameGenerator.generateID();
                        topicID.add(attrValue);

                        final URI tmpId = changeTable.get(setFragment(currentParsingFile, id));
                        if (tmpId != null && tmpId.equals(setFragment(outputFile, id))) {
                            changeTable.put(setFragment(currentParsingFile, id), setFragment(outputFile, attrValue));
                        }

                        final URI tmpVal = changeTable.get(currentParsingFile);
                        if (tmpVal != null && tmpVal.equals(setFragment(outputFile, id))) {
                            changeTable.put(currentParsingFile, setFragment(outputFile, attrValue));
                        }
                        currentParsingFileTopicIDChangeTable.put(oldAttr, attrValue);
                    } else {
                        topicID.add(attrValue);
                    }
                }
            } else if (ATTRIBUTE_NAME_HREF.equals(attrName)) {
                // update @href value
                if (checkHREF(resAtts)) {
                    // if current @href value needs to be updated
                    URI relative = getRelativePath(outputFile, currentParsingFile);
                    if (conflictTable.containsKey(outputFile)) {
                        final URI realoutputfile = conflictTable.get(outputFile);
                        relative = getRelativePath(realoutputfile, currentParsingFile);
                    }
                    if (attrValue.startsWith(SHARP)) {
                        // if @href refers to a location inside current parsing file
                        // update @href to point back to current file
                        // if the location is moved to chunk, @href will
                        // be update again to the new location.
                        attrValue = relative + attrValue;
                    } else if (relative.toString().contains(SLASH)) {
                        // if new file is not under the same directory with current file
                        // add path information to the @href value
                        attrValue = relative.resolve(attrValue).toString();
                    }
                }
            }
            resAtts.setValue(i, attrValue);
        }
        if (TOPIC_TOPIC.matches(cls) && resAtts.getValue(ATTRIBUTE_NAMESPACE_PREFIX_DITAARCHVERSION) == null) {
            addOrSetAttribute(resAtts, ATTRIBUTE_NAMESPACE_PREFIX_DITAARCHVERSION, DITA_NAMESPACE);
        }
        if (TOPIC_TOPIC.matches(cls) && resAtts.getValue(XMLNS_ATTRIBUTE + ":" + DITA_OT_PREFIX) == null) {
            addOrSetAttribute(resAtts, XMLNS_ATTRIBUTE + ":" + DITA_OT_PREFIX, DITA_OT_NS);
        }
        return resAtts;
    }

    /**
     * Generate file name.
     *
     * @return generated file name
     */
    String generateFilename() {
        return chunkFilenameGenerator.generateFilename(CHUNK_PREFIX, FILE_EXTENSION_DITA);
    }

    /**
     * Check whether href needs to be updated
     */
    private boolean checkHREF(final Attributes atts) {
        if (atts.getValue(ATTRIBUTE_NAME_HREF) == null) {
            return false;
        }
        return !ATTR_SCOPE_VALUE_EXTERNAL.equals(atts.getValue(ATTRIBUTE_NAME_SCOPE));
    }

    /**
     * Create topicmeta node.
     *
     * @param topic document element of a topic file.
     * @return created and populated topicmeta
     */<commit_msg>Refactor attribute processing in chunk<commit_after>/*
 * This file is part of the DITA Open Toolkit project.
 * See the accompanying license.txt file for applicable licenses.
 */

/*
 * (c) Copyright IBM Corp. 2007 All Rights Reserved.
 */
package org.dita.dost.writer;

import org.dita.dost.exception.DITAOTException;
import org.dita.dost.module.ChunkModule.ChunkFilenameGenerator;
import org.dita.dost.util.Job.FileInfo;
import org.dita.dost.util.TopicIdParser;
import org.dita.dost.util.XMLUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.Text;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.AttributesImpl;

import java.io.*;
import java.net.URI;
import java.util.*;

import static javax.xml.XMLConstants.XMLNS_ATTRIBUTE;
import static org.dita.dost.reader.ChunkMapReader.*;
import static org.dita.dost.util.Constants.*;
import static org.dita.dost.util.FileUtils.isAbsolutePath;
import static org.dita.dost.util.URLUtils.*;
import static org.dita.dost.util.XMLUtils.*;
import static org.dita.dost.writer.ImageMetadataFilter.DITA_OT_NS;
import static org.dita.dost.writer.ImageMetadataFilter.DITA_OT_PREFIX;

/**
 * ChunkTopicParser class, writing chunking content into relative topic files
 * and then update list. Not reusable and not thread-safe.
 */
public abstract class AbstractChunkTopicParser extends AbstractXMLWriter {

    /**
     * Keys and values are absolute chimera paths, i.e. systems paths with fragments
     */
    LinkedHashMap<URI, URI> changeTable = null;
    /**
     * Keys and values are absolute chimera paths, i.e. systems paths with fragments
     */
    Map<URI, URI> conflictTable = null;

    Element rootTopicref = null;

    /** Input file's parent absolute path. */
    URI currentFile = null;

    URI currentParsingFile = null;
    /** Absolute temporary output file */
    URI outputFile = null;

    String targetTopicId = null;

    String selectMethod = CHUNK_SELECT_DOCUMENT;
    // flag whether output the nested nodes
    boolean include = false;
    boolean skip = false;

    int includelevel = 0;
    int skipLevel = 0;

    final Set<String> topicSpecSet = new HashSet<>(16);

    boolean startFromFirstTopic = false;

    Writer output = null;

    Set<String> topicID = new HashSet<>();

    Map<String, String> currentParsingFileTopicIDChangeTable;

    private ChunkFilenameGenerator chunkFilenameGenerator;

    abstract public void write(final URI filename) throws DITAOTException;

    @Override
    public void write(final File fileDir) throws DITAOTException {
        throw new UnsupportedOperationException();
    }

    /**
     * Set up the class.
     *
     * @param changeTable   changeTable
     * @param conflictTable conflictTable
     * @param rootTopicref  chunking topicref
     */
    public void setup(final LinkedHashMap<URI, URI> changeTable, final Map<URI, URI> conflictTable,
                      final Element rootTopicref,
                      final ChunkFilenameGenerator chunkFilenameGenerator) {
        this.changeTable = changeTable;
        this.rootTopicref = rootTopicref;
        this.conflictTable = conflictTable;
        this.chunkFilenameGenerator = chunkFilenameGenerator;
    }

    // Filter methods

    @Override
    public void characters(final char[] ch, final int start, final int length) throws SAXException {
        if (include) {
            try {
                output.write(escapeXML(ch, start, length));
            } catch (final IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    @Override
    public void endDocument() throws SAXException {
        include = false;
        skip = false;
    }

    @Override
    public abstract void endElement(final String uri, final String localName, final String qName) throws SAXException;

    @Override
    public void ignorableWhitespace(final char[] ch, final int start, final int length) throws SAXException {
        if (include) {
            try {
                output.write(ch, start, length);
            } catch (final IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    @Override
    public void processingInstruction(final String target, final String data) throws SAXException {
        if (include
                || PI_WORKDIR_TARGET.equals(target)
                || PI_WORKDIR_TARGET_URI.equals(target)
                || PI_PATH2PROJ_TARGET.equals(target)
                || PI_PATH2PROJ_TARGET_URI.equals(target)) {
            try {
                writeProcessingInstruction(output, target, data);
            } catch (final IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    @Override
    public void startDocument() throws SAXException {
        // difference between to-content & select-topic
        if (CHUNK_SELECT_DOCUMENT.equals(selectMethod)) {
            // currentParsingFile can never equal outputFile except when
            // chunk="to-content" is set at map level
            if ((currentParsingFile).equals(outputFile)) {
                // if current file serves as root of new chunk
                // include will be set to true in startDocument()
                // in order to copy PIs and <dita> element
                // otherwise, if current file is copied to other file
                // do not copy PIs and <dita>element
                include = true;
                skip = false;
                skipLevel = 0;
            } else {
                include = false;
                startFromFirstTopic = true;
                skip = false;
                skipLevel = 0;
            }
        }
    }

    @Override
    public abstract void startElement(final String uri, final String localName, final String qName, final Attributes atts)
            throws SAXException;

    void processSelect(String id) {
        if (include) {
            if (CHUNK_SELECT_TOPIC.equals(selectMethod)) {
                // if select method is "select-topic" and current topic is the nested topic in target topic, skip it.
                include = false;
                skipLevel = 1;
                skip = true;
            } else {
                // if select method is "select-document" or "select-branch"
                // and current topic is the nested topic in target topic.
                // if file name has been changed, add an entry in changeTable
                if (!currentParsingFile.equals(outputFile)) {
                    if (id != null) {
                        changeTable.put(setFragment(currentParsingFile, id), setFragment(outputFile, id));
                    } else {
                        changeTable.put(stripFragment(currentParsingFile), stripFragment(outputFile));
                    }
                }
            }
        } else if (skip) {
            skipLevel = 1;
        } else if (id != null && (id.equals(targetTopicId) || startFromFirstTopic)) {
            // if the target topic has not been found and current topic is the target topic
            include = true;
            includelevel = 0;
            skip = false;
            skipLevel = 0;
            startFromFirstTopic = false;
            if (!currentParsingFile.equals(outputFile)) {
                changeTable.put(setFragment(currentParsingFile, id), setFragment(outputFile, id));
            }
        }
    }

    FileInfo generateFileInfo(final URI output) {
        final URI temp = job.tempDirURI.relativize(output);
        final URI result = job.getInputDir().resolve(temp);
        final FileInfo.Builder b = currentParsingFile != null
                ? new FileInfo.Builder(job.getFileInfo(stripFragment(currentParsingFile)))
                : new FileInfo.Builder();
        final FileInfo fi = b
                .uri(temp)
                .result(result)
                .format(ATTR_FORMAT_VALUE_DITA)
                .build();
        return fi;
    }

    URI generateOutputFilename(final String id) {
        URI newFileName = currentFile.resolve(id + FILE_EXTENSION_DITA);
        if (id == null || new File(newFileName).exists()) {
            final URI t = newFileName;
            newFileName = currentFile.resolve(generateFilename());
            conflictTable.put(newFileName, t);
        }
        return newFileName;
    }

    Attributes processAttributes(final Attributes atts) {
        final String cls = atts.getValue(ATTRIBUTE_NAME_CLASS);
        final AttributesImpl resAtts = new AttributesImpl(atts);

        final String id = resAtts.getValue(ATTRIBUTE_NAME_ID);
        if (id != null && TOPIC_TOPIC.matches(cls)) {
            if (topicID.contains(id)) {
                final String newId = chunkFilenameGenerator.generateID();
                topicID.add(newId);

                final URI tmpId = changeTable.get(setFragment(currentParsingFile, id));
                if (tmpId != null && tmpId.equals(setFragment(outputFile, id))) {
                    changeTable.put(setFragment(currentParsingFile, id), setFragment(outputFile, newId));
                }

                final URI tmpVal = changeTable.get(currentParsingFile);
                if (tmpVal != null && tmpVal.equals(setFragment(outputFile, id))) {
                    changeTable.put(currentParsingFile, setFragment(outputFile, newId));
                }
                currentParsingFileTopicIDChangeTable.put(id, newId);
                XMLUtils.addOrSetAttribute(resAtts, ATTRIBUTE_NAME_ID, newId);
            } else {
                topicID.add(id);
            }
        }
        final String href = resAtts.getValue(ATTRIBUTE_NAME_HREF);
        final String scope = resAtts.getValue(ATTRIBUTE_NAME_SCOPE);
        if (href != null && !ATTR_SCOPE_VALUE_EXTERNAL.equals(scope)) {
            // if current @href value needs to be updated
            URI relative = getRelativePath(outputFile, currentParsingFile);
            if (conflictTable.containsKey(outputFile)) {
                final URI realoutputfile = conflictTable.get(outputFile);
                relative = getRelativePath(realoutputfile, currentParsingFile);
            }
            if (href.startsWith(SHARP)) {
                // if @href refers to a location inside current parsing file
                // update @href to point back to current file
                // if the location is moved to chunk, @href will
                // be update again to the new location.
                XMLUtils.addOrSetAttribute(resAtts, ATTRIBUTE_NAME_HREF, relative + href);
            } else if (relative.toString().contains(SLASH)) {
                // if new file is not under the same directory with current file
                // add path information to the @href value
                XMLUtils.addOrSetAttribute(resAtts, ATTRIBUTE_NAME_HREF, relative.resolve(href).toString());
            }
        }
        if (TOPIC_TOPIC.matches(cls) && resAtts.getValue(ATTRIBUTE_NAMESPACE_PREFIX_DITAARCHVERSION) == null) {
            addOrSetAttribute(resAtts, ATTRIBUTE_NAMESPACE_PREFIX_DITAARCHVERSION, DITA_NAMESPACE);
        }
        if (TOPIC_TOPIC.matches(cls) && resAtts.getValue(XMLNS_ATTRIBUTE + ":" + DITA_OT_PREFIX) == null) {
            addOrSetAttribute(resAtts, XMLNS_ATTRIBUTE + ":" + DITA_OT_PREFIX, DITA_OT_NS);
        }
        return resAtts;
    }

    /**
     * Generate file name.
     *
     * @return generated file name
     */
    String generateFilename() {
        return chunkFilenameGenerator.generateFilename(CHUNK_PREFIX, FILE_EXTENSION_DITA);
    }

    /**
     * Create topicmeta node.
     *
     * @param topic document element of a topic file.
     * @return created and populated topicmeta
     */
{'fraction_non_alphanumeric': 0.08604670284492683, 'fraction_numerical': 0.0016855780299292879, 'mean_word_length': 2.0493920020057668, 'pattern_counts': {'":': 4, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3435695', 'n_tokens_mistral': 6543, 'n_tokens_neox': 6296, 'n_words': 1721}