<commit_before>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.elasticsearch.hadoop.util;

import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.URL;

import org.apache.commons.codec.binary.Base64;
import org.elasticsearch.hadoop.serialization.SerializationException;

/**
 * Utility class used internally for the Pig support.
 */
public abstract class IOUtils {

    public static String serializeToBase64(Serializable object) throws IOException {
        FastByteArrayOutputStream baos = new FastByteArrayOutputStream();
        ObjectOutputStream oos = new ObjectOutputStream(baos);
        try {
            oos.writeObject(object);
        } finally {
            close(oos);
        }
        return StringUtils.asUTFString(Base64.encodeBase64(baos.bytes().bytes(), false, true));
    }

    @SuppressWarnings("unchecked")
    public static <T extends Serializable> T deserializeFromBase64(String data) {
        byte[] rawData = Base64.decodeBase64(data.getBytes(StringUtils.UTF_8));
        ObjectInputStream ois = null;
        try {
            ois = new ObjectInputStream(new FastByteArrayInputStream(rawData));
            Object o = ois.readObject();
            return (T) o;
        } catch (ClassNotFoundException ex) {
            throw new IllegalStateException("cannot deserialize object", ex);
        } catch (IOException ex) {
            throw new SerializationException("cannot deserialize object", ex);
        } finally {
            close(ois);
        }
    }

    public static BytesArray asBytes(InputStream in) throws IOException {
        return asBytes(new BytesArray(in.available()), in);
    }

    public static BytesArray asBytes(BytesArray ba, InputStream in) throws IOException {
        FastByteArrayOutputStream bos = new FastByteArrayOutputStream(ba);
        byte[] buffer = new byte[1024];
        int read = 0;
        try {
            while ((read = in.read(buffer)) != -1) {
                bos.write(buffer, 0, read);
            }

        } finally {
            try {
                in.close();
            } catch (IOException ex) {<commit_msg>remove usage of Base64 from apache commons<commit_after>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.elasticsearch.hadoop.util;

import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.URL;

import javax.xml.bind.DatatypeConverter;

import org.elasticsearch.hadoop.serialization.SerializationException;

/**
 * Utility class used internally for the Pig support.
 */
public abstract class IOUtils {

    public static String serializeToBase64(Serializable object) throws IOException {
        FastByteArrayOutputStream baos = new FastByteArrayOutputStream();
        ObjectOutputStream oos = new ObjectOutputStream(baos);
        try {
            oos.writeObject(object);
        } finally {
            close(oos);
        }
        return DatatypeConverter.printBase64Binary(baos.bytes().bytes());
    }

    @SuppressWarnings("unchecked")
    public static <T extends Serializable> T deserializeFromBase64(String data) {
        byte[] rawData = DatatypeConverter.parseBase64Binary(data);
        ObjectInputStream ois = null;
        try {
            ois = new ObjectInputStream(new FastByteArrayInputStream(rawData));
            Object o = ois.readObject();
            return (T) o;
        } catch (ClassNotFoundException ex) {
            throw new IllegalStateException("cannot deserialize object", ex);
        } catch (IOException ex) {
            throw new SerializationException("cannot deserialize object", ex);
        } finally {
            close(ois);
        }
    }

    public static BytesArray asBytes(InputStream in) throws IOException {
        return asBytes(new BytesArray(in.available()), in);
    }

    public static BytesArray asBytes(BytesArray ba, InputStream in) throws IOException {
        FastByteArrayOutputStream bos = new FastByteArrayOutputStream(ba);
        byte[] buffer = new byte[1024];
        int read = 0;
        try {
            while ((read = in.read(buffer)) != -1) {
                bos.write(buffer, 0, read);
            }

        } finally {
            try {
                in.close();
            } catch (IOException ex) {
{'fraction_non_alphanumeric': 0.08383534136546185, 'fraction_numerical': 0.00786479250334672, 'mean_word_length': 2.919344262295082, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5560185', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1483, 'n_words': 512}