<commit_before>/*
 * ################################################################
 *
 * ProActive: The Java(TM) library for Parallel, Distributed,
 *            Concurrent computing with Security and Mobility
 *
 * Copyright (C) 1997-2002 INRIA/University of Nice-Sophia Antipolis
 * Contact: <EMAIL>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 *  Initial developer(s):               The ProActive Team
 *                        http://www.inria.fr/oasis/ProActive/contacts.html
 *  Contributor(s):
 *
 * ################################################################
 */
package util;

import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.ext.LexicalHandler;
import org.xml.sax.helpers.DefaultHandler;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

import java.util.HashMap;
import java.util.Map;
import java.util.Vector;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;


/** This works this way:
 * Some kind of ugly sax parsing is done on the xml file, spitting out the tags as they come,
 * until a tag 'programlisting' or 'screen' is found. The data inside the tags is then
 * highlighted by the method highlight(String).  It is then fed back into the stream.
 * Also modifies 'xref' and 'textdata'. */
public class DocBookize extends DefaultHandler implements LexicalHandler {
    static String EOL = System.getProperty("line.separator");
    static boolean SHORT_LINES = false;
    private final String[] filePath; // the possible places where to look for cited files
    private String programContent = null; // when reading programlisting, stores the characters for highlighting
    private BufferedWriter out; // the output stream, ie the initial file + decoration  
    private String outputFileName; // the name of the output file
    private String htmlizedJavaPath;
    private String language = ""; // when storing programlisting, the language in which the code is written

    /** An association between 'language' ==> 'highlighter for language' */
    private Map languageHighlighters = new HashMap(); // type is <String => LanguageToDocBook> 

    /** fields needed to know where to add the source code at the end of the docbook xml */
    private Vector listOFids = null; // type is <String> Vector
    private HashMap srcContentsToAdd = new HashMap(); // type is String ==> StringBuffer
    private Vector srcFilesAlreadyAdded = new Vector(); // type is <String> Vector 

    /** @param fileName The name of the XML file which should have its <screen> tags decorated
     * @param path The paths on which to find the files to include in the docbook */
    public DocBookize(String fileName, String htmlizedJava, String[] path) {
        this.outputFileName = fileName;
        this.htmlizedJavaPath = htmlizedJava;
        this.filePath = path; // the possible places where to look for cited files
        this.languageHighlighters.put("java", new JavaToDocBookRegexp());
        this.languageHighlighters.put("xml", new XmlToDocBookRegexp());
    }

    public static void main(String[] argv) {
        // usage warning
        if (argv.length < 2) {
            System.err.println(
                "Usage: docBookize filename pathForJavaHtmlized [pathForTextData]*");
            System.err.println(
                "Transforms <programlisting> tags (side-effect: removes docbook indentation)");
            System.exit(-1);
        }

        String fileToBeautify = argv[0];
        String htmlizedJava = argv[1];
        System.out.println(
            "[INFO] Beautifying code examples within <programlisting> tags (" +
            fileToBeautify + ")");

        // Create SAX machinery 
        File inputFile = new File(fileToBeautify);

        //path to look for files = argv - (2 first occurrence) + (inputFile Directory)
        String[] path = new String[argv.length - 1];
        path[0] = inputFile.getParent() + "/"; // the directory of file to beautify
        System.arraycopy(argv, 2, path, 1, argv.length - 2);

        DocBookize handler = new DocBookize(fileToBeautify + ".tmp",
                htmlizedJava, path);

        SAXParserFactory factory = SAXParserFactory.newInstance();

        factory.setNamespaceAware(true);

        try {
            SAXParser saxParser = factory.newSAXParser();
            saxParser.getXMLReader()
                     .setProperty("http://xml.org/sax/properties/lexical-handler",
                handler);
            // parse the file, using DocBookize.methods to handle tags
            saxParser.parse(inputFile, handler);

            // rename output file to first name ==> overwrite
            File resultFile = handler.getOutputFile();

            if (!inputFile.delete()) {
                throw new IOException("Could not delete file " + inputFile +
                    ".");
            }

            if (!resultFile.renameTo(inputFile)) {
                throw new IOException("Could not rename file " + resultFile +
                    " to " + inputFile + ".");
            }
        } catch (Throwable t) {
            t.printStackTrace();
        }
    }

    /** Default handler operation when start document is encountered */
    public void startDocument() throws SAXException {
        // Create the output file
        try {
            out = new BufferedWriter(new FileWriter(this.outputFileName));
        } catch (IOException e) {
            throw new SAXException("I/O error opening temp file", e);
        }

        print("<?xml version='1.0' encoding='UTF-8'?>" + EOL);
        print(
            "<!-- DocBookize has been run to highlight keywords in code examples -->" +
            EOL);
    }

    /** Default handler operation when end document is encountered */
    public void endDocument() throws SAXException {
        try {
            out.close();
        } catch (IOException e) {
            throw new SAXException("I/O error closing temp file", e);
        }
    }<commit_msg>Fixed utf8 bug with documentation for windows/mac<commit_after>/*
 * ################################################################
 *
 * ProActive: The Java(TM) library for Parallel, Distributed,
 *            Concurrent computing with Security and Mobility
 *
 * Copyright (C) 1997-2002 INRIA/University of Nice-Sophia Antipolis
 * Contact: <EMAIL>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 *  Initial developer(s):               The ProActive Team
 *                        http://www.inria.fr/oasis/ProActive/contacts.html
 *  Contributor(s):
 *
 * ################################################################
 */
package util;

import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.ext.LexicalHandler;
import org.xml.sax.helpers.DefaultHandler;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;

import java.util.HashMap;
import java.util.Map;
import java.util.Vector;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;


/** This works this way:
 * Some kind of ugly sax parsing is done on the xml file, spitting out the tags as they come,
 * until a tag 'programlisting' or 'screen' is found. The data inside the tags is then
 * highlighted by the method highlight(String).  It is then fed back into the stream.
 * Also modifies 'xref' and 'textdata'. */
public class DocBookize extends DefaultHandler implements LexicalHandler {
    static String EOL = System.getProperty("line.separator");
    static boolean SHORT_LINES = false;
    private final String[] filePath; // the possible places where to look for cited files
    private String programContent = null; // when reading programlisting, stores the characters for highlighting
    private BufferedWriter out; // the output stream, ie the initial file + decoration  
    private String outputFileName; // the name of the output file
    private String htmlizedJavaPath;
    private String language = ""; // when storing programlisting, the language in which the code is written

    /** An association between 'language' ==> 'highlighter for language' */
    private Map languageHighlighters = new HashMap(); // type is <String => LanguageToDocBook> 

    /** fields needed to know where to add the source code at the end of the docbook xml */
    private Vector listOFids = null; // type is <String> Vector
    private HashMap srcContentsToAdd = new HashMap(); // type is String ==> StringBuffer
    private Vector srcFilesAlreadyAdded = new Vector(); // type is <String> Vector 

    /** @param fileName The name of the XML file which should have its <screen> tags decorated
     * @param path The paths on which to find the files to include in the docbook */
    public DocBookize(String fileName, String htmlizedJava, String[] path) {
        this.outputFileName = fileName;
        this.htmlizedJavaPath = htmlizedJava;
        this.filePath = path; // the possible places where to look for cited files
        this.languageHighlighters.put("java", new JavaToDocBookRegexp());
        this.languageHighlighters.put("xml", new XmlToDocBookRegexp());
    }

    public static void main(String[] argv) {
        // usage warning
        if (argv.length < 2) {
            System.err.println(
                "Usage: docBookize filename pathForJavaHtmlized [pathForTextData]*");
            System.err.println(
                "Transforms <programlisting> tags (side-effect: removes docbook indentation)");
            System.exit(-1);
        }

        String fileToBeautify = argv[0];
        String htmlizedJava = argv[1];
        System.out.println(
            "[INFO] Beautifying code examples within <programlisting> tags (" +
            fileToBeautify + ")");

        // Create SAX machinery 
        File inputFile = new File(fileToBeautify);

        //path to look for files = argv - (2 first occurrence) + (inputFile Directory)
        String[] path = new String[argv.length - 1];
        path[0] = inputFile.getParent() + "/"; // the directory of file to beautify
        System.arraycopy(argv, 2, path, 1, argv.length - 2);

        DocBookize handler = new DocBookize(fileToBeautify + ".tmp",
                htmlizedJava, path);

        SAXParserFactory factory = SAXParserFactory.newInstance();

        factory.setNamespaceAware(true);

        try {
            SAXParser saxParser = factory.newSAXParser();
            saxParser.getXMLReader()
                     .setProperty("http://xml.org/sax/properties/lexical-handler",
                handler);
            // parse the file, using DocBookize.methods to handle tags
            saxParser.parse(inputFile, handler);

            // rename output file to first name ==> overwrite
            File resultFile = handler.getOutputFile();

            if (!inputFile.delete()) {
                throw new IOException("Could not delete file " + inputFile +
                    ".");
            }

            if (!resultFile.renameTo(inputFile)) {
                throw new IOException("Could not rename file " + resultFile +
                    " to " + inputFile + ".");
            }
        } catch (Throwable t) {
            t.printStackTrace();
        }
    }

    /** Default handler operation when start document is encountered */
    public void startDocument() throws SAXException {
        // Create the output file
        try {
            out = new BufferedWriter(new OutputStreamWriter(
                        new FileOutputStream(this.outputFileName), "UTF8"));
        } catch (IOException e) {
            throw new SAXException("I/O error opening temp file", e);
        }

        print("<?xml version='1.0' encoding='UTF-8'?>" + EOL);
        print(
            "<!-- DocBookize has been run to highlight keywords in code examples -->" +
            EOL);
    }

    /** Default handler operation when end document is encountered */
    public void endDocument() throws SAXException {
        try {
            out.close();
        } catch (IOException e) {
            throw new SAXException("I/O error closing temp file", e);
        }
    }
{'fraction_non_alphanumeric': 0.10845561528403312, 'fraction_numerical': 0.005520848467238123, 'mean_word_length': 2.6800320769847636, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 2, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5253125', 'n_tokens_mistral': 3605, 'n_tokens_neox': 3448, 'n_words': 1293}