<commit_before>/*******************************************************************************
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *******************************************************************************/
package org.ofbiz.base.util.template;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;

import org.ofbiz.base.location.FlexibleLocation;
import org.ofbiz.base.util.Debug;
import org.ofbiz.base.util.StringUtil;
import org.ofbiz.base.util.UtilCodec;
import org.ofbiz.base.util.UtilGenerics;
import org.ofbiz.base.util.UtilMisc;
import org.ofbiz.base.util.UtilProperties;
import org.ofbiz.base.util.UtilRender;
import org.ofbiz.base.util.UtilValidate;
import org.ofbiz.base.util.cache.UtilCache;

import freemarker.cache.TemplateLoader;
import freemarker.core.Environment;
import freemarker.ext.beans.BeanModel;
import freemarker.ext.beans.BeansWrapper;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
import freemarker.template.ObjectWrapper;
import freemarker.template.SimpleHash;
import freemarker.template.SimpleScalar;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import freemarker.template.TemplateHashModel;
import freemarker.template.TemplateModel;
import freemarker.template.TemplateModelException;
import freemarker.template.Version;
import freemarker.template.utility.DeepUnwrap;

/**
 * FreeMarkerWorker - Freemarker Template Engine Utilities.
 * <p>
 * SCIPIO: 2017-04-03: All ObjectWrappers are now made from custom types in ScipioFtlWrappers and
 * they support custom plugin wrapping logic.
 */
public final class FreeMarkerWorker {

    private static final Debug.OfbizLogger module = Debug.getOfbizLogger(java.lang.invoke.MethodHandles.lookup().lookupClass());

    public static final Version version = Configuration.VERSION_2_3_28;

    private FreeMarkerWorker () {}

    // use soft references for this so that things from Content records don't kill all of our memory, or maybe not for performance reasons... hmmm, leave to config file...
    private static final UtilCache<String, Template> cachedTemplates = UtilCache.createUtilCache("template.ftl.general", 0, 0, false);
    // SCIPIO: 2017-04-03: custom wrapper FIXME: should not force BeansWrapper in the future...
    //private static final BeansWrapper defaultOfbizWrapper = new BeansWrapperBuilder(version).build();
    private static final BeansWrapper defaultOfbizWrapper = (BeansWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultOfbizWrapper(version);
    private static final Configuration defaultOfbizConfig = makeConfiguration(defaultOfbizWrapper);

    /**
     * SCIPIO: The default escaping charset for the Freemarker <code>?url</code> built-in.
     * <p>
     * NOTE: This is hardcoded here because it is also already hardcoded in <code>UrlCodec</code>.
     */
    private static final String defaultUrlEscapingCharset = "UTF-8";

    /**
     * SCIPIO: A version of defaultOfbizWrapper that produces simple maps (SimpleMapAdapter).
     * FIXME: should not force BeansWrapper in the future...
     */
    private static final BeansWrapper defaultOfbizSimpleMapWrapper = (BeansWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultOfbizSimpleMapWrapper(version);

    /**
     * SCIPIO: A basic object wrapper that produces mainly simple, inline-FTL-like types,
     * using adapters for collections.
     * FIXME: should not force DefaultObjectWrapper in the future...
     */
    private static final DefaultObjectWrapper defaultSimpleTypeWrapper = (DefaultObjectWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultSimpleTypeWrapper(version);

    /**
     * SCIPIO: A basic object wrapper that produces mainly simple, inline-FTL-like types,
     * using copies for collections.
     * FIXME: should not force DefaultObjectWrapper in the future...
     */
    private static final DefaultObjectWrapper defaultSimpleTypeCopyingWrapper = (DefaultObjectWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultSimpleTypeCopyingWrapper(version);

    /**
     * SCIPIO: A copy of the current thread Environment.
     * @see #getCurrentEnvironment
     */
    private static final ThreadLocal<Environment> threadEnv = new ThreadLocal<Environment>();

    public static BeansWrapper getDefaultOfbizWrapper() {
        return defaultOfbizWrapper;
    }

    /**
     * SCIPIO: Get version of getDefaultOfbizWrapper that is the same but produces simple maps.
     */
    public static ObjectWrapper getDefaultOfbizSimpleMapWrapper() {
        return defaultOfbizSimpleMapWrapper;
    }

    /**
     * SCIPIO: Get a Freemarker simple type wrapper.
     */
    public static ObjectWrapper getDefaultSimpleTypeWrapper() {
        return defaultSimpleTypeWrapper;
    }

    /**
     * SCIPIO: Get a Freemarker simple type copying wrapper.
     */
    public static ObjectWrapper getDefaultSimpleTypeCopyingWrapper() {
        return defaultSimpleTypeCopyingWrapper;
    }

    public static Configuration newConfiguration() {
        return new Configuration(version);
    }

    public static Configuration makeConfiguration(BeansWrapper wrapper) {
        Configuration newConfig = newConfiguration();

        newConfig.setObjectWrapper(wrapper);
        TemplateHashModel staticModels = wrapper.getStaticModels();
        newConfig.setSharedVariable("Static", staticModels);
        try {
            newConfig.setSharedVariable("EntityQuery", staticModels.get("org.ofbiz.entity.util.EntityQuery"));
        } catch (TemplateModelException e) {
            Debug.logError(e, module);
        }
        newConfig.setLocalizedLookup(false);
        newConfig.setSharedVariable("StringUtil", new BeanModel(StringUtil.INSTANCE, wrapper));
        newConfig.setTemplateLoader(new FlexibleTemplateLoader());
        newConfig.setAutoImports(UtilProperties.getProperties("freemarkerImports"));

        // SCIPIO: New code for includes and shared vars...
        Properties includeProperties = UtilProperties.getProperties("freemarkerIncludes");
        Properties sharedVarsProperties = UtilProperties.getProperties("freemarkerSharedVars");
        loadSharedVars(sharedVarsProperties,newConfig);
        List<Object> includeFreemarkerTemplates = new ArrayList<Object>(includeProperties.values());
        if (includeFreemarkerTemplates.size() > 0) {
            newConfig.setAutoIncludes(includeFreemarkerTemplates);
        }
        // ... end.

        newConfig.setTemplateExceptionHandler(new FreeMarkerWorker.OFBizTemplateExceptionHandler());
        try {
            newConfig.setSetting("datetime_format", "yyyy-MM-dd HH:mm:ss.SSS");
            newConfig.setSetting("number_format", "0.##########");<commit_msg>stock patch: freemarkerImports.properties: now loads from any component<commit_after>/*******************************************************************************
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *******************************************************************************/
package org.ofbiz.base.util.template;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;

import org.ofbiz.base.location.FlexibleLocation;
import org.ofbiz.base.util.Debug;
import org.ofbiz.base.util.StringUtil;
import org.ofbiz.base.util.UtilCodec;
import org.ofbiz.base.util.UtilGenerics;
import org.ofbiz.base.util.UtilMisc;
import org.ofbiz.base.util.UtilProperties;
import org.ofbiz.base.util.UtilRender;
import org.ofbiz.base.util.UtilValidate;
import org.ofbiz.base.util.cache.UtilCache;

import freemarker.cache.TemplateLoader;
import freemarker.core.Environment;
import freemarker.ext.beans.BeanModel;
import freemarker.ext.beans.BeansWrapper;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
import freemarker.template.ObjectWrapper;
import freemarker.template.SimpleHash;
import freemarker.template.SimpleScalar;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import freemarker.template.TemplateHashModel;
import freemarker.template.TemplateModel;
import freemarker.template.TemplateModelException;
import freemarker.template.Version;
import freemarker.template.utility.DeepUnwrap;

/**
 * FreeMarkerWorker - Freemarker Template Engine Utilities.
 * <p>
 * SCIPIO: 2017-04-03: All ObjectWrappers are now made from custom types in ScipioFtlWrappers and
 * they support custom plugin wrapping logic.
 */
public final class FreeMarkerWorker {

    private static final Debug.OfbizLogger module = Debug.getOfbizLogger(java.lang.invoke.MethodHandles.lookup().lookupClass());

    public static final Version version = Configuration.VERSION_2_3_28;

    private FreeMarkerWorker () {}

    // use soft references for this so that things from Content records don't kill all of our memory, or maybe not for performance reasons... hmmm, leave to config file...
    private static final UtilCache<String, Template> cachedTemplates = UtilCache.createUtilCache("template.ftl.general", 0, 0, false);
    // SCIPIO: 2017-04-03: custom wrapper FIXME: should not force BeansWrapper in the future...
    //private static final BeansWrapper defaultOfbizWrapper = new BeansWrapperBuilder(version).build();
    private static final BeansWrapper defaultOfbizWrapper = (BeansWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultOfbizWrapper(version);
    private static final Configuration defaultOfbizConfig = makeConfiguration(defaultOfbizWrapper);

    /**
     * SCIPIO: The default escaping charset for the Freemarker <code>?url</code> built-in.
     * <p>
     * NOTE: This is hardcoded here because it is also already hardcoded in <code>UrlCodec</code>.
     */
    private static final String defaultUrlEscapingCharset = "UTF-8";

    /**
     * SCIPIO: A version of defaultOfbizWrapper that produces simple maps (SimpleMapAdapter).
     * FIXME: should not force BeansWrapper in the future...
     */
    private static final BeansWrapper defaultOfbizSimpleMapWrapper = (BeansWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultOfbizSimpleMapWrapper(version);

    /**
     * SCIPIO: A basic object wrapper that produces mainly simple, inline-FTL-like types,
     * using adapters for collections.
     * FIXME: should not force DefaultObjectWrapper in the future...
     */
    private static final DefaultObjectWrapper defaultSimpleTypeWrapper = (DefaultObjectWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultSimpleTypeWrapper(version);

    /**
     * SCIPIO: A basic object wrapper that produces mainly simple, inline-FTL-like types,
     * using copies for collections.
     * FIXME: should not force DefaultObjectWrapper in the future...
     */
    private static final DefaultObjectWrapper defaultSimpleTypeCopyingWrapper = (DefaultObjectWrapper) ScipioFtlWrappers.getSystemObjectWrapperFactory().getDefaultSimpleTypeCopyingWrapper(version);

    /**
     * SCIPIO: A copy of the current thread Environment.
     * @see #getCurrentEnvironment
     */
    private static final ThreadLocal<Environment> threadEnv = new ThreadLocal<Environment>();

    public static BeansWrapper getDefaultOfbizWrapper() {
        return defaultOfbizWrapper;
    }

    /**
     * SCIPIO: Get version of getDefaultOfbizWrapper that is the same but produces simple maps.
     */
    public static ObjectWrapper getDefaultOfbizSimpleMapWrapper() {
        return defaultOfbizSimpleMapWrapper;
    }

    /**
     * SCIPIO: Get a Freemarker simple type wrapper.
     */
    public static ObjectWrapper getDefaultSimpleTypeWrapper() {
        return defaultSimpleTypeWrapper;
    }

    /**
     * SCIPIO: Get a Freemarker simple type copying wrapper.
     */
    public static ObjectWrapper getDefaultSimpleTypeCopyingWrapper() {
        return defaultSimpleTypeCopyingWrapper;
    }

    public static Configuration newConfiguration() {
        return new Configuration(version);
    }

    public static Configuration makeConfiguration(BeansWrapper wrapper) {
        Configuration newConfig = newConfiguration();

        newConfig.setObjectWrapper(wrapper);
        TemplateHashModel staticModels = wrapper.getStaticModels();
        newConfig.setSharedVariable("Static", staticModels);
        try {
            newConfig.setSharedVariable("EntityQuery", staticModels.get("org.ofbiz.entity.util.EntityQuery"));
        } catch (TemplateModelException e) {
            Debug.logError(e, module);
        }
        newConfig.setLocalizedLookup(false);
        newConfig.setSharedVariable("StringUtil", new BeanModel(StringUtil.INSTANCE, wrapper));
        newConfig.setTemplateLoader(new FlexibleTemplateLoader());
        // SCIPIO: Load it from ALL components, like freemarkerTransforms:
        //newConfig.setAutoImports(UtilProperties.getProperties("freemarkerImports"));
        newConfig.setAutoImports(UtilProperties.getMergedPropertiesFromAllComponents("freemarkerImports"));
        
        // SCIPIO: New code for includes and shared vars...
        Properties includeProperties = UtilProperties.getMergedPropertiesFromAllComponents("freemarkerIncludes");
        Properties sharedVarsProperties = UtilProperties.getMergedPropertiesFromAllComponents("freemarkerSharedVars");
        loadSharedVars(sharedVarsProperties,newConfig);
        List<Object> includeFreemarkerTemplates = new ArrayList<Object>(includeProperties.values());
        if (includeFreemarkerTemplates.size() > 0) {
            newConfig.setAutoIncludes(includeFreemarkerTemplates);
        }
        // ... end.

        newConfig.setTemplateExceptionHandler(new FreeMarkerWorker.OFBizTemplateExceptionHandler());
        try {
            newConfig.setSetting("datetime_format", "yyyy-MM-dd HH:mm:ss.SSS");
            newConfig.setSetting("number_format", "0.##########");
{'fraction_non_alphanumeric': 0.10254414845854534, 'fraction_numerical': 0.003472014366956001, 'mean_word_length': 4.397738287560582, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '533936', 'n_tokens_mistral': 4403, 'n_tokens_neox': 4280, 'n_words': 1181}