<commit_before>/*
 * This file is part of the DITA Open Toolkit project hosted on
 * Sourceforge.net. See the accompanying license.txt file for 
 * applicable licenses.
 */

/*
 * (c) Copyright IBM Corp. 2004, 2005 All Rights Reserved.
 */
package org.dita.dost.reader;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;

import org.dita.dost.exception.DITAOTXMLErrorHandler;
import org.dita.dost.log.DITAOTJavaLogger;
import org.dita.dost.log.MessageUtils;
import org.dita.dost.module.Content;
import org.dita.dost.module.ContentImpl;
import org.dita.dost.resolver.DitaURIResolverFactory;
import org.dita.dost.resolver.URIResolverAdapter;
import org.dita.dost.util.Constants;
import org.dita.dost.util.FileUtils;
import org.dita.dost.util.StringUtils;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;


/**
 * MapLinksReader reads and parse the index information. It is also used to parse
 * map link information in "maplinks.unordered" file.
 * 
 * NOTE: Result of this reader is a map organized as HashMap<String, HashMap<String, String> >.
 * Logically it is organized as:
 * 
 * 		topic-file-path --+-- topic-id-1 --+-- index-entry-1-1
 * 		                  |                +-- index-entry-1-2
 *                        |                +-- ...
 *                        |                +-- index-entry-1-n
 *                        |
 *                        +-- topic-id-1 --+-- ...
 *                        +-- ...
 *                        +-- topic-id-m --+-- ...
 * 
 * IF the input URL DOES NOT specify a topic ID explicitly, we use the special character '#' to
 * stand for the first topic ID this parser encounters.
 * 
 * @author Zhang, <NAME>
 */
public class MapLinksReader extends AbstractXMLReader {
    private static final String INTERNET_LINK_MARK = "://";

    // check whether the index entries we got is meaningfull and valid
    private static boolean verifyIndexEntries(StringBuffer str) {
    	int start;
    	int end;
    	String temp;
        if (str.length() == 0) {
            return false;
        }
        start = str.indexOf(Constants.GREATER_THAN); // start from first tag's end
        end = str.lastIndexOf(Constants.LESS_THAN); // end at last tag's start

        /*
         * original code check whether there is text between different tags
         * modified code check whether there is any content between first and
         * last tags
         */
                
        temp = str.substring(start + 1, end);
        if (temp.trim().length() != 0) {
            return true;
        }
        return false;
    }
    private List<String> ancestorList;
    private String filePath = null;
    private String firstMatchElement;
    private StringBuffer indexEntries;
    private File inputFile;
    private HashSet<String> lastMatchElement;
    private int level;
    private DITAOTJavaLogger logger;
    private HashMap<String, HashMap<String,String> > map;
    private boolean match;

    /*
     * meta shows whether the event is in metadata when using sax to parse
     * ditmap file.
     */
    private List<String> matchList;
    private boolean needResolveEntity;
    private XMLReader reader;
    private String topicPath;
    private boolean validHref;// whether the current href target is internal dita topic file
    

    /**
     * Default constructor of MapLinksReader class.
     */
    public MapLinksReader() {
        super();
        map = new HashMap<String, HashMap<String,String> >();
        ancestorList = new ArrayList<String>(Constants.INT_16);
        matchList = new ArrayList<String>(Constants.INT_16);
        indexEntries = new StringBuffer(Constants.INT_1024);
        firstMatchElement = null;
        lastMatchElement = new HashSet<String>();
        level = 0;
        match = false;
        validHref = true;
        needResolveEntity = false;
        topicPath = null;
        inputFile = null; 
        logger = new DITAOTJavaLogger();
        
        
        try {
            reader = StringUtils.getXMLReader();
            reader.setContentHandler(this);
            reader.setProperty(Constants.LEXICAL_HANDLER_PROPERTY,this);
            //Added by william on 2009-11-8 for ampbug:2893664 start
			reader.setFeature("http://apache.org/xml/features/scanner/notify-char-refs", true);
			reader.setFeature("http://apache.org/xml/features/scanner/notify-builtin-refs", true);
			//Added by william on 2009-11-8 for ampbug:2893664 end
        } catch (Exception e) {
        	logger.logException(e);
        }

    }

    /**
     * @see org.xml.sax.ContentHandler#characters(char[], int, int)
     * 
     */
    public void characters(char[] ch, int start, int length)
            throws SAXException {
<commit_msg>MapLinksReader should not be namespace aware - ID: 3189883<commit_after>/*
 * This file is part of the DITA Open Toolkit project hosted on
 * Sourceforge.net. See the accompanying license.txt file for 
 * applicable licenses.
 */

/*
 * (c) Copyright IBM Corp. 2004, 2005 All Rights Reserved.
 */
package org.dita.dost.reader;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;

import org.dita.dost.exception.DITAOTXMLErrorHandler;
import org.dita.dost.log.DITAOTJavaLogger;
import org.dita.dost.log.MessageUtils;
import org.dita.dost.module.Content;
import org.dita.dost.module.ContentImpl;
import org.dita.dost.resolver.DitaURIResolverFactory;
import org.dita.dost.resolver.URIResolverAdapter;
import org.dita.dost.util.Constants;
import org.dita.dost.util.FileUtils;
import org.dita.dost.util.StringUtils;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;


/**
 * MapLinksReader reads and parse the index information. It is also used to parse
 * map link information in "maplinks.unordered" file.
 * 
 * NOTE: Result of this reader is a map organized as HashMap<String, HashMap<String, String> >.
 * Logically it is organized as:
 * 
 * 		topic-file-path --+-- topic-id-1 --+-- index-entry-1-1
 * 		                  |                +-- index-entry-1-2
 *                        |                +-- ...
 *                        |                +-- index-entry-1-n
 *                        |
 *                        +-- topic-id-1 --+-- ...
 *                        +-- ...
 *                        +-- topic-id-m --+-- ...
 * 
 * IF the input URL DOES NOT specify a topic ID explicitly, we use the special character '#' to
 * stand for the first topic ID this parser encounters.
 * 
 * @author Zhang, <NAME>
 */
public class MapLinksReader extends AbstractXMLReader {
    private static final String INTERNET_LINK_MARK = "://";

    // check whether the index entries we got is meaningfull and valid
    private static boolean verifyIndexEntries(StringBuffer str) {
    	int start;
    	int end;
    	String temp;
        if (str.length() == 0) {
            return false;
        }
        start = str.indexOf(Constants.GREATER_THAN); // start from first tag's end
        end = str.lastIndexOf(Constants.LESS_THAN); // end at last tag's start

        /*
         * original code check whether there is text between different tags
         * modified code check whether there is any content between first and
         * last tags
         */
                
        temp = str.substring(start + 1, end);
        if (temp.trim().length() != 0) {
            return true;
        }
        return false;
    }
    private List<String> ancestorList;
    private String filePath = null;
    private String firstMatchElement;
    private StringBuffer indexEntries;
    private File inputFile;
    private HashSet<String> lastMatchElement;
    private int level;
    private DITAOTJavaLogger logger;
    private HashMap<String, HashMap<String,String> > map;
    private boolean match;

    /*
     * meta shows whether the event is in metadata when using sax to parse
     * ditmap file.
     */
    private List<String> matchList;
    private boolean needResolveEntity;
    private XMLReader reader;
    private String topicPath;
    private boolean validHref;// whether the current href target is internal dita topic file
    

    /**
     * Default constructor of MapLinksReader class.
     */
    public MapLinksReader() {
        super();
        map = new HashMap<String, HashMap<String,String> >();
        ancestorList = new ArrayList<String>(Constants.INT_16);
        matchList = new ArrayList<String>(Constants.INT_16);
        indexEntries = new StringBuffer(Constants.INT_1024);
        firstMatchElement = null;
        lastMatchElement = new HashSet<String>();
        level = 0;
        match = false;
        validHref = true;
        needResolveEntity = false;
        topicPath = null;
        inputFile = null; 
        logger = new DITAOTJavaLogger();
        
        
        try {
            reader = StringUtils.getXMLReader();
            reader.setContentHandler(this);
            reader.setProperty(Constants.LEXICAL_HANDLER_PROPERTY,this);
            //Added by william on 2009-11-8 for ampbug:2893664 start
			reader.setFeature("http://apache.org/xml/features/scanner/notify-char-refs", true);
			reader.setFeature("http://apache.org/xml/features/scanner/notify-builtin-refs", true);
			//Added by william on 2009-11-8 for ampbug:2893664 end
			reader.setFeature("http://xml.org/sax/features/namespaces", false);
        } catch (Exception e) {
        	logger.logException(e);
        }

    }

    /**
     * @see org.xml.sax.ContentHandler#characters(char[], int, int)
     * 
     */
    public void characters(char[] ch, int start, int length)
            throws SAXException {

{'fraction_non_alphanumeric': 0.10285255122539172, 'fraction_numerical': 0.011751707513057453, 'mean_word_length': 2.305776892430279, 'pattern_counts': {'":': 2, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7477562', 'n_tokens_mistral': 2970, 'n_tokens_neox': 2800, 'n_words': 787}