<commit_before>/*
 * Copyright 2003-2016 MarkLogic Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.marklogic.contentpump.utilities;

import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;

import com.marklogic.contentpump.ContentWithFileNameWritable;
import com.marklogic.contentpump.DatabaseDocumentWithMeta;
import com.marklogic.contentpump.RDFWritable;
import com.marklogic.contentpump.TransformOutputFormat;
import com.marklogic.io.Base64;
import com.marklogic.mapreduce.ContentType;
import com.marklogic.xcc.AdhocQuery;
import com.marklogic.xcc.ContentCapability;
import com.marklogic.xcc.ContentCreateOptions;
import com.marklogic.xcc.ContentPermission;
import com.marklogic.xcc.DocumentRepairLevel;
import com.marklogic.xcc.types.ValueType;

/**
 * Helper class for server-side transform
 * @author ali
 *
 */
public class TransformHelper {
    public static final Log LOG = LogFactory.getLog(TransformHelper.class);
    private static String MAP_ELEM_START_TAG = 
        "<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi"
        + "=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:map=\"http:"
        + "//marklogic.com/xdmp/map\">";

    private static String getInvokeModuleQuery(String moduleUri, String functionNs, String functionName,
        String functionParam) {
        StringBuilder q = new StringBuilder();
        q.append("xquery version \"1.0-ml\";\n")
            .append("import module namespace hadoop = \"http://marklogic.com")
            .append("/xdmp/hadoop\" at \"/MarkLogic/hadoop.xqy\";\n")
            .append("declare variable $URI as xs:string external;\n")
            .append("declare variable $CONTENT as item() external;\n")
            .append(
                "declare variable $INSERT-OPTIONS as element() external;\n")
            .append("hadoop:transform-and-insert(\"").append(moduleUri)
            .append("\",\"").append(functionNs).append("\",\"")
            .append(functionName).append("\",\"").append(functionParam)
            .append("\", $URI, $CONTENT, $INSERT-OPTIONS)");
        return q.toString();
    }

    private static String getTypeFromMap(String uri) {
        int idx = uri.lastIndexOf(".");
        Text format = null;
        if (idx != -1) {
            String suff = uri.substring(idx + 1, uri.length());
            if (suff.equalsIgnoreCase("xml"))
                return "xml";
            format = (Text) TransformOutputFormat.mimetypeMap.get(new Text(suff));
        }
        if (format == null) {
            return "binary";
        } else {
            return format.toString();
        }<commit_msg>bug:37763 escape double quote in transform_param<commit_after>/*
 * Copyright 2003-2016 MarkLogic Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.marklogic.contentpump.utilities;

import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;

import com.marklogic.contentpump.ContentWithFileNameWritable;
import com.marklogic.contentpump.DatabaseDocumentWithMeta;
import com.marklogic.contentpump.RDFWritable;
import com.marklogic.contentpump.TransformOutputFormat;
import com.marklogic.io.Base64;
import com.marklogic.mapreduce.ContentType;
import com.marklogic.xcc.AdhocQuery;
import com.marklogic.xcc.ContentCapability;
import com.marklogic.xcc.ContentCreateOptions;
import com.marklogic.xcc.ContentPermission;
import com.marklogic.xcc.DocumentRepairLevel;
import com.marklogic.xcc.types.ValueType;

/**
 * Helper class for server-side transform
 * @author ali
 *
 */
public class TransformHelper {
    public static final Log LOG = LogFactory.getLog(TransformHelper.class);
    private static String MAP_ELEM_START_TAG = 
        "<map:map xmlns:xs=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi"
        + "=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:map=\"http:"
        + "//marklogic.com/xdmp/map\">";

    private static String getInvokeModuleQuery(String moduleUri, String functionNs, String functionName,
        String functionParam) {
        StringBuilder q = new StringBuilder();
        q.append("xquery version \"1.0-ml\";\n")
            .append("import module namespace hadoop = \"http://marklogic.com")
            .append("/xdmp/hadoop\" at \"/MarkLogic/hadoop.xqy\";\n")
            .append("declare variable $URI as xs:string external;\n")
            .append("declare variable $CONTENT as item() external;\n")
            .append(
                "declare variable $INSERT-OPTIONS as element() external;\n")
            .append("hadoop:transform-and-insert(\"").append(moduleUri)
            .append("\",\"").append(functionNs).append("\",\"")
            .append(functionName).append("\",\"")
            .append(functionParam.replace("\"", "\"\""))
            .append("\", $URI, $CONTENT, $INSERT-OPTIONS)");
        return q.toString();
    }

    private static String getTypeFromMap(String uri) {
        int idx = uri.lastIndexOf(".");
        Text format = null;
        if (idx != -1) {
            String suff = uri.substring(idx + 1, uri.length());
            if (suff.equalsIgnoreCase("xml"))
                return "xml";
            format = (Text) TransformOutputFormat.mimetypeMap.get(new Text(suff));
        }
        if (format == null) {
            return "binary";
        } else {
            return format.toString();
        }
{'fraction_non_alphanumeric': 0.12507212925562608, 'fraction_numerical': 0.008799769186381997, 'mean_word_length': 3.196731234866828, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6722819', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1948, 'n_words': 490}