<commit_before>/**
 * Copyright 2016 LinkedIn Corp. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
package com.github.ambry.utils;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


/**
 * A set of utility methods
 */
public class Utils {

  /**
   * Constant to define "infinite" time.
   * <p/>
   * Currently used in lieu of either an epoch based ms expiration time or a seconds based TTL (relative to creation
   * time).
   */
  public static final long Infinite_Time = -1;
  /**
   * The lowest possible port number.
   */
  public static final int MIN_PORT_NUM = 1;
  /**
   * The highest possible port number.
   */
  public static final int MAX_PORT_NUM = 65535;
  private static final String CLIENT_RESET_EXCEPTION_MSG = "Connection reset by peer";
  private static final String CLIENT_BROKEN_PIPE_EXCEPTION_MSG = "Broken pipe";
  private static final Logger logger = LoggerFactory.getLogger(Utils.class);

  // The read*String methods assume that the underlying stream is blocking

  /**
   * Reads a String whose length is a short from the given input stream
   * @param input The input stream from which to read the String from
   * @return The String read from the stream
   * @throws IOException
   */
  public static String readShortString(DataInputStream input) throws IOException {
    Short size = input.readShort();
    if (size < 0) {
      throw new IllegalArgumentException("readShortString : the size cannot be negative");
    }
    byte[] bytes = new byte[size];
    int read = 0;
    while (read < size) {
      int readBytes = input.read(bytes, read, size - read);
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readShortString : the size of the input does not match the actual data size");
    }
    return new String(bytes, "UTF-8");
  }

  /**
   * Gets the size of the string in serialized form
   * @param value the string of interest to be serialized
   * @return the size of the string in serialized form
   */
  public static int getIntStringLength(String value) {
    return value == null ? Integer.BYTES : Integer.BYTES + value.length();
  }

  /**
   * Reads a String whose length is an int from the given input stream
   * @param input The input stream from which to read the String from
   * @return The String read from the stream
   * @throws IOException
   */
  public static String readIntString(DataInputStream input) throws IOException {
    return readIntString(input, StandardCharsets.UTF_8);
  }

  /**
   * Reads a String whose length is an int from the given input stream
   * @param input The input stream from which to read the String from
   * @param charset the charset to use.
   * @return The String read from the stream
   * @throws IOException
   */
  public static String readIntString(DataInputStream input, Charset charset) throws IOException {
    int size = input.readInt();
    if (size < 0) {
      throw new IllegalArgumentException("readIntString : the size cannot be negative");
    }
    byte[] bytes = new byte[size];
    int read = 0;
    while (read < size) {
      int readBytes = input.read(bytes, read, size - read);
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readIntString : the size of the input does not match the actual data size");
    }
    return new String(bytes, charset);
  }

  /**
   *
   * @param input
   * @return
   * @throws IOException
   */
  public static ByteBuffer readIntBuffer(DataInputStream input) throws IOException {
    int size = input.readInt();
    if (size < 0) {
      throw new IllegalArgumentException("readIntBuffer : the size cannot be negative");
    }
    ByteBuffer buffer = ByteBuffer.allocate(size);
    int read = 0;
    while (read < size) {
      int readBytes = input.read(buffer.array());
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readIntBuffer : the size of the input does not match the actual data size");
    }
    return buffer;
  }

  /**
   *
   * @param input
   * @return
   * @throws IOException
   */
  public static ByteBuffer readShortBuffer(DataInputStream input) throws IOException {
    short size = input.readShort();
    if (size < 0) {
      throw new IllegalArgumentException("readShortBuffer : the size cannot be negative");
    }
    ByteBuffer buffer = ByteBuffer.allocate(size);
    int read = 0;
    while (read < size) {
      int readBytes = input.read(buffer.array());
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readShortBuffer the size of the input does not match the actual data size");
    }
    return buffer;
  }

  /**
   * Create a new thread
   *
   * @param runnable The work for the thread to do
   * @param daemon Should the thread block JVM shutdown?
   * @return The unstarted thread
   */
  public static Thread newThread(Runnable runnable, boolean daemon) {
    Thread thread = new Thread(runnable);
    thread.setDaemon(daemon);
    thread.setUncaughtExceptionHandler((t, e) -> {
      logger.error("Encountered throwable in {}", t, e);
    });
    return thread;
  }

  /**
   * Create a new thread
   *
   * @param name The name of the thread
   * @param runnable The work for the thread to do
   * @param daemon Should the thread block JVM shutdown?
   * @return The unstarted thread
   */
  public static Thread newThread(String name, Runnable runnable, boolean daemon) {
    Thread thread = new Thread(runnable, name);
    thread.setDaemon(daemon);
    thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
      public void uncaughtException(Thread t, Throwable e) {
        e.printStackTrace();
      }
    });
    return thread;
  }

  /**
   * Create a daemon thread
   *
   * @param runnable The runnable to execute in the background
   * @return The unstarted thread<commit_msg>Log thread death reasons instead of printing (#843)<commit_after>/**
 * Copyright 2016 LinkedIn Corp. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
package com.github.ambry.utils;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


/**
 * A set of utility methods
 */
public class Utils {

  /**
   * Constant to define "infinite" time.
   * <p/>
   * Currently used in lieu of either an epoch based ms expiration time or a seconds based TTL (relative to creation
   * time).
   */
  public static final long Infinite_Time = -1;
  /**
   * The lowest possible port number.
   */
  public static final int MIN_PORT_NUM = 1;
  /**
   * The highest possible port number.
   */
  public static final int MAX_PORT_NUM = 65535;
  private static final String CLIENT_RESET_EXCEPTION_MSG = "Connection reset by peer";
  private static final String CLIENT_BROKEN_PIPE_EXCEPTION_MSG = "Broken pipe";
  private static final Logger logger = LoggerFactory.getLogger(Utils.class);

  // The read*String methods assume that the underlying stream is blocking

  /**
   * Reads a String whose length is a short from the given input stream
   * @param input The input stream from which to read the String from
   * @return The String read from the stream
   * @throws IOException
   */
  public static String readShortString(DataInputStream input) throws IOException {
    Short size = input.readShort();
    if (size < 0) {
      throw new IllegalArgumentException("readShortString : the size cannot be negative");
    }
    byte[] bytes = new byte[size];
    int read = 0;
    while (read < size) {
      int readBytes = input.read(bytes, read, size - read);
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readShortString : the size of the input does not match the actual data size");
    }
    return new String(bytes, "UTF-8");
  }

  /**
   * Gets the size of the string in serialized form
   * @param value the string of interest to be serialized
   * @return the size of the string in serialized form
   */
  public static int getIntStringLength(String value) {
    return value == null ? Integer.BYTES : Integer.BYTES + value.length();
  }

  /**
   * Reads a String whose length is an int from the given input stream
   * @param input The input stream from which to read the String from
   * @return The String read from the stream
   * @throws IOException
   */
  public static String readIntString(DataInputStream input) throws IOException {
    return readIntString(input, StandardCharsets.UTF_8);
  }

  /**
   * Reads a String whose length is an int from the given input stream
   * @param input The input stream from which to read the String from
   * @param charset the charset to use.
   * @return The String read from the stream
   * @throws IOException
   */
  public static String readIntString(DataInputStream input, Charset charset) throws IOException {
    int size = input.readInt();
    if (size < 0) {
      throw new IllegalArgumentException("readIntString : the size cannot be negative");
    }
    byte[] bytes = new byte[size];
    int read = 0;
    while (read < size) {
      int readBytes = input.read(bytes, read, size - read);
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readIntString : the size of the input does not match the actual data size");
    }
    return new String(bytes, charset);
  }

  /**
   *
   * @param input
   * @return
   * @throws IOException
   */
  public static ByteBuffer readIntBuffer(DataInputStream input) throws IOException {
    int size = input.readInt();
    if (size < 0) {
      throw new IllegalArgumentException("readIntBuffer : the size cannot be negative");
    }
    ByteBuffer buffer = ByteBuffer.allocate(size);
    int read = 0;
    while (read < size) {
      int readBytes = input.read(buffer.array());
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readIntBuffer : the size of the input does not match the actual data size");
    }
    return buffer;
  }

  /**
   *
   * @param input
   * @return
   * @throws IOException
   */
  public static ByteBuffer readShortBuffer(DataInputStream input) throws IOException {
    short size = input.readShort();
    if (size < 0) {
      throw new IllegalArgumentException("readShortBuffer : the size cannot be negative");
    }
    ByteBuffer buffer = ByteBuffer.allocate(size);
    int read = 0;
    while (read < size) {
      int readBytes = input.read(buffer.array());
      if (readBytes == -1 || readBytes == 0) {
        break;
      }
      read += readBytes;
    }
    if (read != size) {
      throw new IllegalArgumentException("readShortBuffer the size of the input does not match the actual data size");
    }
    return buffer;
  }

  /**
   * Create a new thread
   *
   * @param runnable The work for the thread to do
   * @param daemon Should the thread block JVM shutdown?
   * @return The unstarted thread
   */
  public static Thread newThread(Runnable runnable, boolean daemon) {
    Thread thread = new Thread(runnable);
    thread.setDaemon(daemon);
    thread.setUncaughtExceptionHandler((t, e) -> {
      logger.error("Encountered throwable in {}", t, e);
    });
    return thread;
  }

  /**
   * Create a new thread
   *
   * @param name The name of the thread
   * @param runnable The work for the thread to do
   * @param daemon Should the thread block JVM shutdown?
   * @return The unstarted thread
   */
  public static Thread newThread(String name, Runnable runnable, boolean daemon) {
    Thread thread = new Thread(runnable, name);
    thread.setDaemon(daemon);
    thread.setUncaughtExceptionHandler((t, e) -> {
      logger.error("Encountered throwable in {}", t, e);
    });
    return thread;
  }

  /**
   * Create a daemon thread
   *
   * @param runnable The runnable to execute in the background
   * @return The unstarted thread
{'fraction_non_alphanumeric': 0.09352839711237462, 'fraction_numerical': 0.004663642752188079, 'mean_word_length': 3.2468800868149756, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2889698', 'n_tokens_mistral': 4465, 'n_tokens_neox': 4349, 'n_words': 1588}