<commit_before>/*
 * Licensed to ElasticSearch and <NAME> under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.common.xcontent;

import com.google.common.base.Charsets;
import com.google.common.collect.Maps;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.lzf.LZF;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.CachedStreamInput;
import org.elasticsearch.common.io.stream.LZFStreamInput;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;

/**
 *
 */
public class XContentHelper {

    public static XContentParser createParser(byte[] data, int offset, int length) throws IOException {
        if (LZF.isCompressed(data, offset, length)) {
            BytesStreamInput siBytes = new BytesStreamInput(data, offset, length, false);
            LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes);
            XContentType contentType = XContentFactory.xContentType(siLzf);
            siLzf.resetToBufferStart();
            return XContentFactory.xContent(contentType).createParser(siLzf);
        } else {
            return XContentFactory.xContent(data, offset, length).createParser(data, offset, length);
        }
    }

    public static Tuple<XContentType, Map<String, Object>> convertToMap(byte[] data, int offset, int length, boolean ordered) throws ElasticSearchParseException {
        try {
            XContentParser parser;
            XContentType contentType;
            if (LZF.isCompressed(data, offset, length)) {
                BytesStreamInput siBytes = new BytesStreamInput(data, offset, length, false);
                LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes);
                contentType = XContentFactory.xContentType(siLzf);
                siLzf.resetToBufferStart();
                parser = XContentFactory.xContent(contentType).createParser(siLzf);
            } else {
                contentType = XContentFactory.xContentType(data, offset, length);
                parser = XContentFactory.xContent(contentType).createParser(data, offset, length);
            }
            if (ordered) {
                return Tuple.tuple(contentType, parser.mapOrderedAndClose());
            } else {
                return Tuple.tuple(contentType, parser.mapAndClose());
            }
        } catch (IOException e) {
            throw new ElasticSearchParseException("Failed to parse content to map", e);
        }
    }

    public static String convertToJson(byte[] data, int offset, int length, boolean reformatJson) throws IOException {
        XContentType xContentType = XContentFactory.xContentType(data, offset, length);
        if (xContentType == XContentType.JSON && reformatJson) {
            return new String(data, offset, length, Charsets.UTF_8);
        }
        XContentParser parser = null;
        try {
            parser = XContentFactory.xContent(xContentType).createParser(data, offset, length);
            parser.nextToken();
            XContentBuilder builder = XContentFactory.jsonBuilder();
            builder.copyCurrentStructure(parser);
            return builder.string();
        } finally {
            if (parser != null) {
                parser.close();
            }
        }
    }

    /**
     * Merges the defaults provided as the second parameter into the content of the first. Only does recursive merge
     * for inner maps.
     */
    @SuppressWarnings({"unchecked"})
    public static void mergeDefaults(Map<String, Object> content, Map<String, Object> defaults) {
        for (Map.Entry<String, Object> defaultEntry : defaults.entrySet()) {
            if (!content.containsKey(defaultEntry.getKey())) {
                // copy it over, it does not exists in the content
                content.put(defaultEntry.getKey(), defaultEntry.getValue());
            } else {
                // in the content and in the default, only merge compound ones (maps)
                if (content.get(defaultEntry.getKey()) instanceof Map && defaultEntry.getValue() instanceof Map) {
                    mergeDefaults((Map<String, Object>) content.get(defaultEntry.getKey()), (Map<String, Object>) defaultEntry.getValue());
                } else if (content.get(defaultEntry.getKey()) instanceof List && defaultEntry.getValue() instanceof List) {
                    List defaultList = (List) defaultEntry.getValue();
                    List contentList = (List) content.get(defaultEntry.getKey());
<commit_msg>convert to json wasn't taking reformat flag properly into account<commit_after>/*
 * Licensed to ElasticSearch and <NAME> under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.common.xcontent;

import com.google.common.base.Charsets;
import com.google.common.collect.Maps;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.lzf.LZF;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.CachedStreamInput;
import org.elasticsearch.common.io.stream.LZFStreamInput;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;

/**
 *
 */
public class XContentHelper {

    public static XContentParser createParser(byte[] data, int offset, int length) throws IOException {
        if (LZF.isCompressed(data, offset, length)) {
            BytesStreamInput siBytes = new BytesStreamInput(data, offset, length, false);
            LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes);
            XContentType contentType = XContentFactory.xContentType(siLzf);
            siLzf.resetToBufferStart();
            return XContentFactory.xContent(contentType).createParser(siLzf);
        } else {
            return XContentFactory.xContent(data, offset, length).createParser(data, offset, length);
        }
    }

    public static Tuple<XContentType, Map<String, Object>> convertToMap(byte[] data, int offset, int length, boolean ordered) throws ElasticSearchParseException {
        try {
            XContentParser parser;
            XContentType contentType;
            if (LZF.isCompressed(data, offset, length)) {
                BytesStreamInput siBytes = new BytesStreamInput(data, offset, length, false);
                LZFStreamInput siLzf = CachedStreamInput.cachedLzf(siBytes);
                contentType = XContentFactory.xContentType(siLzf);
                siLzf.resetToBufferStart();
                parser = XContentFactory.xContent(contentType).createParser(siLzf);
            } else {
                contentType = XContentFactory.xContentType(data, offset, length);
                parser = XContentFactory.xContent(contentType).createParser(data, offset, length);
            }
            if (ordered) {
                return Tuple.tuple(contentType, parser.mapOrderedAndClose());
            } else {
                return Tuple.tuple(contentType, parser.mapAndClose());
            }
        } catch (IOException e) {
            throw new ElasticSearchParseException("Failed to parse content to map", e);
        }
    }

    public static String convertToJson(byte[] data, int offset, int length, boolean reformatJson) throws IOException {
        return convertToJson(data, offset, length, reformatJson, false);
    }

    public static String convertToJson(byte[] data, int offset, int length, boolean reformatJson, boolean prettyPrint) throws IOException {
        XContentType xContentType = XContentFactory.xContentType(data, offset, length);
        if (xContentType == XContentType.JSON && !reformatJson) {
            return new String(data, offset, length, Charsets.UTF_8);
        }
        XContentParser parser = null;
        try {
            parser = XContentFactory.xContent(xContentType).createParser(data, offset, length);
            parser.nextToken();
            XContentBuilder builder = XContentFactory.jsonBuilder();
            if (prettyPrint) {
                builder.prettyPrint();
            }
            builder.copyCurrentStructure(parser);
            return builder.string();
        } finally {
            if (parser != null) {
                parser.close();
            }
        }
    }

    /**
     * Merges the defaults provided as the second parameter into the content of the first. Only does recursive merge
     * for inner maps.
     */
    @SuppressWarnings({"unchecked"})
    public static void mergeDefaults(Map<String, Object> content, Map<String, Object> defaults) {
        for (Map.Entry<String, Object> defaultEntry : defaults.entrySet()) {
            if (!content.containsKey(defaultEntry.getKey())) {
                // copy it over, it does not exists in the content
                content.put(defaultEntry.getKey(), defaultEntry.getValue());
            } else {
                // in the content and in the default, only merge compound ones (maps)
                if (content.get(defaultEntry.getKey()) instanceof Map && defaultEntry.getValue() instanceof Map) {
                    mergeDefaults((Map<String, Object>) content.get(defaultEntry.getKey()), (Map<String, Object>) defaultEntry.getValue());
                } else if (content.get(defaultEntry.getKey()) instanceof List && defaultEntry.getValue() instanceof List) {
                    List defaultList = (List) defaultEntry.getValue();
                    List contentList = (List) content.get(defaultEntry.getKey());

{'fraction_non_alphanumeric': 0.09207322571918117, 'fraction_numerical': 0.0009017945711966813, 'mean_word_length': 2.8480222068008327, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6961430', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2803, 'n_words': 809}