<commit_before>/*
 The contents of this file are subject to the Mozilla Public License
 Version 1.1 (the "License"); you may not use this file except in
 compliance with the License. You may obtain a copy of the License at
 http://www.mozilla.org/MPL/

 Software distributed under the License is distributed on an "AS IS" basis,
 WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
 the specific language governing rights and limitations under the License.

 The Original Code is TEAM Engine.

 The Initial Developer of the Original Code is Northrop Grumman Corporation
 jointly with The National Technology Alliance.  Portions created by
 Northrop Grumman Corporation are Copyright (C) 2005-2006, Northrop
 Grumman Corporation. All Rights Reserved.

 Contributor(s): No additional contributors to date
 */

package com.occamlab.te;

import java.io.File;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.xml.XMLConstants;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;

import net.sf.saxon.FeatureKeys;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.Serializer;
import net.sf.saxon.s9api.XdmAtomicValue;
import net.sf.saxon.s9api.XsltCompiler;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;

import com.occamlab.te.index.Index;
import com.occamlab.te.util.Misc;

public class Generator {
    // Generates XSL template files from CTL sources and a master index
    // of metadata about the CTL objects
    private static Logger logger = Logger.getLogger("com.occamlab.te.Generator");

    public static Index generateXsl(SetupOptions opts) throws Exception {
        Index masterIndex = new Index();
        
        // Create CTL validator
        SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
        Schema ctl_schema = sf.newSchema(Misc.getResourceAsFile("com/occamlab/te/schemas/ctl.xsd"));
        Validator ctl_validator = ctl_schema.newValidator();
        CtlErrorHandler validation_eh = new CtlErrorHandler();
        ctl_validator.setErrorHandler(validation_eh);
        
        // Create a transformer to generate executable scripts from CTL sources
        Processor processor = new Processor(false);
        processor.setConfigurationProperty(FeatureKeys.XINCLUDE, Boolean.TRUE);
        processor.setConfigurationProperty(FeatureKeys.LINE_NUMBERING, Boolean.TRUE);
        XsltCompiler generatorCompiler = processor.newXsltCompiler();
        File generatorStylesheet = Misc.getResourceAsFile("com/occamlab/te/generate_xsl.xsl");
        XsltExecutable generatorXsltExecutable = generatorCompiler.compile(new StreamSource(generatorStylesheet));
        XsltTransformer generatorTransformer = generatorXsltExecutable.load();

        // Create a list of CTL sources (may be files or dirs)
        ArrayList<File> sources = new ArrayList<File>();
        File f = Misc.getResourceAsFile("com/occamlab/te/scripts/parsers.ctl");
        sources.add(f.getParentFile());
        sources.addAll(opts.getSources());

        // Create a list of source CTL files only (no dirs),
        // and a corresponding list containing a working dir for each file
        ArrayList<File> sourceFiles = new ArrayList<File>(); 
        ArrayList<File> workDirs = new ArrayList<File>();
        Iterator<File> it = sources.iterator();
        while (it.hasNext()) {
            File source = it.next();
//System.out.println("Processing source(s) at: " + source.getAbsolutePath());
//          appLogger.log(Level.INFO, "Processing source(s) at: " + source.getAbsolutePath());

            String encodedName = URLEncoder.encode(source.getAbsolutePath(), "UTF-8");
            encodedName = encodedName.replace('%', '~');  // In Java 5, the Document.parse function has trouble with the URL % encoding
            File workingDir = new File(opts.getWorkDir(), encodedName);
            workingDir.mkdir();
            
            if (source.isDirectory()) {
                String[] children = source.list();
                for (int i = 0; i < children.length; i++) {
                    // Finds all .ctl and .xml files in the directory to use
                    String lowerName = children[i].toLowerCase();
                    if (lowerName.endsWith(".ctl") || lowerName.endsWith(".xml")) {
                        File file = new File(source, children[i]);
                        if (file.isFile()) {
                            sourceFiles.add(file);
                            String basename = children[i].substring(0, children[i].length() - 4);
                            File subdir = new File(workingDir, basename);
                            subdir.mkdir();
                            workDirs.add(subdir);
                        }
                    }<commit_msg>Commented fixed specific for jdk 1.5 when parsing URL with '%' encodings<commit_after>/*
 The contents of this file are subject to the Mozilla Public License
 Version 1.1 (the "License"); you may not use this file except in
 compliance with the License. You may obtain a copy of the License at
 http://www.mozilla.org/MPL/

 Software distributed under the License is distributed on an "AS IS" basis,
 WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
 the specific language governing rights and limitations under the License.

 The Original Code is TEAM Engine.

 The Initial Developer of the Original Code is Northrop Grumman Corporation
 jointly with The National Technology Alliance.  Portions created by
 Northrop Grumman Corporation are Copyright (C) 2005-2006, Northrop
 Grumman Corporation. All Rights Reserved.

 Contributor(s): No additional contributors to date
 */

package com.occamlab.te;

import java.io.File;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.xml.XMLConstants;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;

import net.sf.saxon.FeatureKeys;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.QName;
import net.sf.saxon.s9api.Serializer;
import net.sf.saxon.s9api.XdmAtomicValue;
import net.sf.saxon.s9api.XsltCompiler;
import net.sf.saxon.s9api.XsltExecutable;
import net.sf.saxon.s9api.XsltTransformer;

import com.occamlab.te.index.Index;
import com.occamlab.te.util.Misc;

public class Generator {
    // Generates XSL template files from CTL sources and a master index
    // of metadata about the CTL objects
    private static Logger logger = Logger.getLogger("com.occamlab.te.Generator");

    public static Index generateXsl(SetupOptions opts) throws Exception {
        Index masterIndex = new Index();
        
        // Create CTL validator
        SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
        Schema ctl_schema = sf.newSchema(Misc.getResourceAsFile("com/occamlab/te/schemas/ctl.xsd"));
        Validator ctl_validator = ctl_schema.newValidator();
        CtlErrorHandler validation_eh = new CtlErrorHandler();
        ctl_validator.setErrorHandler(validation_eh);
        
        // Create a transformer to generate executable scripts from CTL sources
        Processor processor = new Processor(false);
        processor.setConfigurationProperty(FeatureKeys.XINCLUDE, Boolean.TRUE);
        processor.setConfigurationProperty(FeatureKeys.LINE_NUMBERING, Boolean.TRUE);
        XsltCompiler generatorCompiler = processor.newXsltCompiler();
        File generatorStylesheet = Misc.getResourceAsFile("com/occamlab/te/generate_xsl.xsl");
        XsltExecutable generatorXsltExecutable = generatorCompiler.compile(new StreamSource(generatorStylesheet));
        XsltTransformer generatorTransformer = generatorXsltExecutable.load();

        // Create a list of CTL sources (may be files or dirs)
        ArrayList<File> sources = new ArrayList<File>();
        File f = Misc.getResourceAsFile("com/occamlab/te/scripts/parsers.ctl");
        sources.add(f.getParentFile());
        sources.addAll(opts.getSources());

        // Create a list of source CTL files only (no dirs),
        // and a corresponding list containing a working dir for each file
        ArrayList<File> sourceFiles = new ArrayList<File>(); 
        ArrayList<File> workDirs = new ArrayList<File>();
        Iterator<File> it = sources.iterator();
        while (it.hasNext()) {
            File source = it.next();
//System.out.println("Processing source(s) at: " + source.getAbsolutePath());
//          appLogger.log(Level.INFO, "Processing source(s) at: " + source.getAbsolutePath());

            String encodedName = URLEncoder.encode(source.getAbsolutePath(), "UTF-8");
           // encodedName = encodedName.replace('%', '~');  // In Java 5, the Document.parse function has trouble with the URL % encoding
            File workingDir = new File(opts.getWorkDir(), encodedName);
            workingDir.mkdir();
            
            if (source.isDirectory()) {
                String[] children = source.list();
                for (int i = 0; i < children.length; i++) {
                    // Finds all .ctl and .xml files in the directory to use
                    String lowerName = children[i].toLowerCase();
                    if (lowerName.endsWith(".ctl") || lowerName.endsWith(".xml")) {
                        File file = new File(source, children[i]);
                        if (file.isFile()) {
                            sourceFiles.add(file);
                            String basename = children[i].substring(0, children[i].length() - 4);
                            File subdir = new File(workingDir, basename);
                            subdir.mkdir();
                            workDirs.add(subdir);
                        }
                    }
{'fraction_non_alphanumeric': 0.09613683279268652, 'fraction_numerical': 0.004718372161604247, 'mean_word_length': 2.855248200075786, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '249499', 'n_tokens_mistral': 2796, 'n_tokens_neox': 2678, 'n_words': 782}