<commit_before>package ecologylab.bigsemantics;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Charsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;

import ecologylab.generic.StringBuilderBaseUtils;
import ecologylab.serialization.SIMPLTranslationException;
import ecologylab.serialization.SimplTypesScope;
import ecologylab.serialization.formatenums.StringFormat;

/**
 * Utility functions.
 * 
 * @author quyin
 */
public class Utils
{

  private static Logger       logger;

  private static HashFunction fingerprintHashFunc;

  private static HashFunction secureHashFunc;

  private static BaseEncoding base64Encoder;

  private static final int    BUF_SIZE = 1024;

  static
  {
    logger = LoggerFactory.getLogger(Utils.class);
    fingerprintHashFunc = Hashing.goodFastHash(32);
    secureHashFunc = Hashing.sha256();
    base64Encoder = BaseEncoding.base64Url();
  }

  /**
   * Get the fingerprint of an input string. A fingerprint is not secure hash.
   * 
   * @param s
   * @return
   */
  public static byte[] fingerprintBytes(String s)
  {
    return fingerprintHashFunc.hashString(s).asBytes();
  }

  /**
   * Get a secure hash of the input string.
   * 
   * @param s
   * @return
   */
  public static byte[] secureHashBytes(String s)
  {
    return secureHashFunc.hashString(s, Charsets.UTF_8).asBytes();
  }

  /**
   * Encode the input bytes in base64.
   * 
   * @param bytes
   * @return
   */
  public static String base64urlEncode(byte[] bytes)
  {
    return base64Encoder.encode(bytes);
  }

  public static String serializeToString(Object obj, StringFormat format)
  {
    try
    {
      return SimplTypesScope.serialize(obj, format).toString();
    }
    catch (SIMPLTranslationException e)
    {
      logger.error("Cannot serialize " + obj, e);
    }
    return "ERROR";
  }

  public static Charset getCharsetByName(String charsetName, Charset defaultCharset)
  {<commit_msg>Fixed NPE.<commit_after>package ecologylab.bigsemantics;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Charsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;

import ecologylab.generic.StringBuilderBaseUtils;
import ecologylab.serialization.SIMPLTranslationException;
import ecologylab.serialization.SimplTypesScope;
import ecologylab.serialization.formatenums.StringFormat;

/**
 * Utility functions.
 * 
 * @author quyin
 */
public class Utils
{

  private static Logger       logger;

  private static HashFunction fingerprintHashFunc;

  private static HashFunction secureHashFunc;

  private static BaseEncoding base64Encoder;

  private static final int    BUF_SIZE = 1024;

  static
  {
    logger = LoggerFactory.getLogger(Utils.class);
    fingerprintHashFunc = Hashing.goodFastHash(32);
    secureHashFunc = Hashing.sha256();
    base64Encoder = BaseEncoding.base64Url();
  }

  /**
   * Get the fingerprint of an input string. A fingerprint is not secure hash.
   * 
   * @param s
   * @return
   */
  public static byte[] fingerprintBytes(String s)
  {
    return fingerprintHashFunc.hashString(s).asBytes();
  }

  /**
   * Get a secure hash of the input string.
   * 
   * @param s
   * @return
   */
  public static byte[] secureHashBytes(String s)
  {
    return secureHashFunc.hashString(s, Charsets.UTF_8).asBytes();
  }

  /**
   * Encode the input bytes in base64.
   * 
   * @param bytes
   * @return
   */
  public static String base64urlEncode(byte[] bytes)
  {
    return base64Encoder.encode(bytes);
  }

  public static String serializeToString(Object obj, StringFormat format)
  {
    try
    {
      return SimplTypesScope.serialize(obj, format).toString();
    }
    catch (SIMPLTranslationException e)
    {
      logger.error("Cannot serialize " + obj, e);
    }
    return "ERROR";
  }

  public static Charset getCharsetByName(String charsetName, Charset defaultCharset)
  {
    if (charsetName == null)
    {
      charsetName = "UTF8";
    }
{'fraction_non_alphanumeric': 0.09717937417364478, 'fraction_numerical': 0.010797708241516086, 'mean_word_length': 3.6601642710472277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '673290', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1388, 'n_words': 324}