<commit_before>package org.wiztools.restclient;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.apache.commons.codec.binary.Base64;
import org.wiztools.commons.Charsets;
import org.wiztools.commons.MultiValueMap;
import org.wiztools.commons.StringUtil;

/**
 *
 * @author schandran
 */
public final class Util {
    
    // private constructor so that no instance from outside can be created
    private Util(){}
    
    public static String base64encode(String inStr) {
        return base64encode(inStr.getBytes(Charsets.UTF_8));
    }
    
    public static String base64encode(byte[] arr) {
        return Base64.encodeBase64String(arr);
    }
    
    public static byte[] base64decodeByteArray(String base64Str) throws Base64Exception {
        if(!Base64.isBase64(base64Str)) {
            throw new Base64Exception("Provided string is not Base64 encoded");
        }
        byte[] out = Base64.decodeBase64(base64Str);
        return out;
    }
    
    public static String base64decode(String base64Str) throws Base64Exception {
        if(!Base64.isBase64(base64Str)) {
            throw new Base64Exception("Provided string is not Base64 encoded");
        }
        byte[] out = base64decodeByteArray(base64Str);
        CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
        try {
            decoder.onMalformedInput(CodingErrorAction.REPORT);
            decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
            CharBuffer buffer = decoder.decode(
                    ByteBuffer.wrap(Arrays.copyOf(out, out.length)));
            return buffer.toString();
        }
        catch(MalformedInputException ex) {
            throw new Base64Exception("Input is malformed", ex);
        }
        catch(UnmappableCharacterException ex) {
            throw new Base64Exception("Unmappable characters found", ex);
        }
        catch(CharacterCodingException ex) {
            throw new Base64Exception(ex);
        }
    }

    public static String getStackTrace(final Throwable aThrowable) {
        String errorMsg = aThrowable.getMessage();
        final Writer result = new StringWriter();
        final PrintWriter printWriter = new PrintWriter(result);
        aThrowable.printStackTrace(printWriter);
        return errorMsg + "\n" + result.toString();
    }

    public static String getHTMLListFromList(List<String> ll) {
        StringBuilder sb = new StringBuilder();
        sb.append("<html><ul>");
        for (String str : ll) {
            sb.append("<li>").append(str).append("</li>");
        }
        sb.append("</ul></html>");
        return sb.toString();
    }
    
    private static final String ENCODE = "UTF-8";
<commit_msg>utility method to get contenttype<commit_after>package org.wiztools.restclient;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.apache.commons.codec.binary.Base64;
import org.wiztools.commons.Charsets;
import org.wiztools.commons.MultiValueMap;
import org.wiztools.commons.StringUtil;

/**
 *
 * @author schandran
 */
public final class Util {
    
    // private constructor so that no instance from outside can be created
    private Util(){}
    
    public static String base64encode(String inStr) {
        return base64encode(inStr.getBytes(Charsets.UTF_8));
    }
    
    public static String base64encode(byte[] arr) {
        return Base64.encodeBase64String(arr);
    }
    
    public static byte[] base64decodeByteArray(String base64Str) throws Base64Exception {
        if(!Base64.isBase64(base64Str)) {
            throw new Base64Exception("Provided string is not Base64 encoded");
        }
        byte[] out = Base64.decodeBase64(base64Str);
        return out;
    }
    
    public static String base64decode(String base64Str) throws Base64Exception {
        if(!Base64.isBase64(base64Str)) {
            throw new Base64Exception("Provided string is not Base64 encoded");
        }
        byte[] out = base64decodeByteArray(base64Str);
        CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
        try {
            decoder.onMalformedInput(CodingErrorAction.REPORT);
            decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
            CharBuffer buffer = decoder.decode(
                    ByteBuffer.wrap(Arrays.copyOf(out, out.length)));
            return buffer.toString();
        }
        catch(MalformedInputException ex) {
            throw new Base64Exception("Input is malformed", ex);
        }
        catch(UnmappableCharacterException ex) {
            throw new Base64Exception("Unmappable characters found", ex);
        }
        catch(CharacterCodingException ex) {
            throw new Base64Exception(ex);
        }
    }
    
    public static ContentType getContentType(String header) {
        Pattern p = Pattern.compile("([^;]+);\\s*charset=([^;]+)");
        Matcher m = p.matcher(header);
        if(m.matches()) {
            String contentType = m.group(1);
            Charset charset = Charset.forName(m.group(2));
            return new ContentTypeBean(contentType, charset);
        }
        return null;
    }
    
    public static ContentType getContentType(MultiValueMap<String, String> headers) {
        String ct;
        String cs;
        for(String key: headers.keySet()) {
            if("content-type".equalsIgnoreCase(key.trim())) {
                return getContentType(headers.get(key).iterator().next());
            }
        }
        return null;
    }

    public static String getStackTrace(final Throwable aThrowable) {
        String errorMsg = aThrowable.getMessage();
        final Writer result = new StringWriter();
        final PrintWriter printWriter = new PrintWriter(result);
        aThrowable.printStackTrace(printWriter);
        return errorMsg + "\n" + result.toString();
    }

    public static String getHTMLListFromList(List<String> ll) {
        StringBuilder sb = new StringBuilder();
        sb.append("<html><ul>");
        for (String str : ll) {
            sb.append("<li>").append(str).append("</li>");
        }
        sb.append("</ul></html>");
        return sb.toString();
    }
    
    private static final String ENCODE = "UTF-8";

{'fraction_non_alphanumeric': 0.10543766578249338, 'fraction_numerical': 0.016976127320954906, 'mean_word_length': 2.927604166666667, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6613486', 'n_tokens_mistral': 2184, 'n_tokens_neox': 2114, 'n_words': 472}