<commit_before>package water.util;

import java.io.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Linux /proc file reader.
 *
 * Read tick information for the system and the current process in order to provide
 * stats on the cloud page about CPU utilization.
 *
 * Tick counts are monotonically increasing since boot.
 *
 * Find definitions of /proc file info here.
 * http://man7.org/linux/man-pages/man5/proc.5.html
 */
public abstract class LinuxProcFileReader {
  private static final char[] buffer = new char[8 * 1024]; // MUST be first in file, or else clinit ordering problems
  // Read all of a file, throwing IOE on buffer-full or returning a String
  private static String readFile(File f) throws IOException {
    int bytesRead = 0;
    try(FileReader fr = new FileReader(f)) {
        synchronized(buffer) {
          while (true) {
            int n = fr.read(buffer, bytesRead, buffer.length - bytesRead);
            if( n < 0 ) return new String(buffer, 0, bytesRead);
            bytesRead += n;
            if( bytesRead >= buffer.length ) throw new IOException("LinuxProcFileReader readFile unexpected buffer full");
          }
        }
    }
  }

  // --------- Process ID --------------
  /** @return process id for this node  */
  public static int getProcessID() { return PID; }
  private static final int PID;
  static { PID = getProcessIdFromJMX(); }
  private static int getProcessIdFromJMX() {
    // Note: may fail in some JVM implementations therefore fallback has to be provided.
    // Something like '<pid>@<hostname>', at least in SUN / Oracle JVMs
    final String jvmName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
    final int index = jvmName.indexOf('@');
    return index < 1 ? -1 // part before '@' empty (index = 0) / '@' not found (index = -1)
      : Integer.parseInt(jvmName.substring(0, index));
  }

  // --------- /proc/pid/status --- Number of available processors
  /** @return number of CPUs allowed by this process. */
  public static char getProcessCpusAllowed() { return PROCESS_CPUS_ALLOWED; }
  private static final char PROCESS_CPUS_ALLOWED;
  static { PROCESS_CPUS_ALLOWED = parseProcessStatusFile(readProcessStatusFile(PID)); }
  private static String readProcessStatusFile(int pid) {
    try { return pid== -1 ? null : readFile(new File("/proc/"+pid+"/status")); }
    catch( IOException _ ) { return null; }
  }
  private static char parseProcessStatusFile(String s) {
    char cpus = (char) Runtime.getRuntime().availableProcessors();
    if(s == null) return cpus;
    try {
      Matcher m = Pattern.compile("Cpus_allowed:\\s+([A-Fa-f0-9,]+)").matcher(s);
      return m.find() ? numSetBitsHex(m.group(1)) : cpus;
    }
    catch( Exception ignore ) { return cpus; }
  }
  /** @return number of set bits in hexadecimal string (chars must be 0-F) */
  private static char numSetBitsHex(String s) {
    // Look-up table for num set bits in 4-bit char
    final int[] bits_set = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
    char nset = 0;
    for(int i = 0; i < s.length(); i++) {
      Character ch = s.charAt(i);
      if (ch == ',') continue;
      int x = Integer.parseInt(ch.toString(), 16);
      nset += bits_set[x];
    }
    return nset;
  }


  // --------- /proc/pid/proc --- process RSS and TotalTicks ----
  /** @return ticks this process was running. */
  public static long getProcessTotalTicks() { return _processTotalTicks; }
  private static long _processTotalTicks = -1;

  /** @return resident set size (RSS) of this process. */
  public static long getProcessRss() { return _processRss; }
  private static long _processRss = -1;

  private static String readProcessProcFile(int pid) {
    try { return pid == -1 ? null : readFile(new File("/proc/"+pid+"/stat")); }
    catch( IOException _ ) { return null; }
  }
  private static final Pattern procPattern = Pattern.compile(
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + ".*");
  private static void parseProcessProcFile(String s) {
    if (s == null) return;
    try {
      BufferedReader reader = new BufferedReader(new StringReader(s));
      Matcher m = procPattern.matcher(reader.readLine());
      if(!m.matches() ) return;
      long processUserTicks   = Long.parseLong(m.group(14));
      long processSystemTicks = Long.parseLong(m.group(15));
      _processTotalTicks = processUserTicks + processSystemTicks;
      _processRss = Long.parseLong(m.group(24));
    }
    catch (Exception ignore) {}
  }


  // --------- /proc/pid/stat --- System and Process cpu ticks -----------
  /**  Array of ticks:  [cpu number][tick type]
   *
   * tick types are:
   * [0] user ticks
   * [1] system ticks
   * [2] other ticks (i/o)
   * [3] idle ticks
   *
   * @return ticks array for each cpu of the system.
   */
  public static long[][] getCpuTicks() { return CPUTICKS; }
  private static final long[][] CPUTICKS = new long[PROCESS_CPUS_ALLOWED][4/*{user,sys,io,idle}*/];
  /** @return ticks the system was idle.  in general:  idle + busy == 100% */
  public static long getSystemIdleTicks() { return _systemIdleTicks; }
  private static long _systemIdleTicks = -1;
  /** @return ticks the system was up. */
  public static long getSystemTotalTicks() { return _systemTotalTicks; }
  private static long _systemTotalTicks = -1;
  // Read from various /proc files, returning a String or null if not available
  private static String readSystemProcFile() {
    try { return readFile(new File("/proc/stat")); }
    catch( IOException _ ) { return null; }
  }
  private static final Pattern sysTicksPattern = Pattern.compile("cpu\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+).*");
  private static final Pattern cpuTicksPattern = Pattern.compile("cpu(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+).*");
  /** @param s String containing contents of proc file. */
  private static void parseSystemProcFile(String s) {
    if (s == null) return;
    BufferedReader reader = new BufferedReader(new StringReader(s));
    try { 
      // Read aggregate cpu values
      String line = reader.readLine();
      Matcher m = sysTicksPattern.matcher(line);
      if( !m.matches() ) return;
  
      int matchGroupBase = 0;
      long systemUserTicks = 0;
      long systemSystemTicks = 0;
      long systemOtherTicks = 0;
      systemUserTicks    += Long.parseLong(m.group(1 + matchGroupBase));
      systemOtherTicks   += Long.parseLong(m.group(2 + matchGroupBase));
      systemSystemTicks  += Long.parseLong(m.group(3 + matchGroupBase));
      _systemIdleTicks    = Long.parseLong(m.group(4 + matchGroupBase));
      systemOtherTicks   += Long.parseLong(m.group(5 + matchGroupBase));
      systemSystemTicks  += Long.parseLong(m.group(6 + matchGroupBase));
      systemSystemTicks  += Long.parseLong(m.group(7 + matchGroupBase));
      _systemTotalTicks   = systemUserTicks + systemOtherTicks + systemSystemTicks + _systemIdleTicks;
  
      // Read individual cpu values
      int cpu = 0;
      line = reader.readLine();
      while (line != null) {
        m = cpuTicksPattern.matcher(line);
        if( !m.matches() ) break;
        // Copying algorithm from http://gee.cs.oswego.edu/dl/code/
        // See perfbar.c in gtk_perfbar package.
        // int cpuNum = Integer.parseInt(m.group(1));
        matchGroupBase = 1;
        long cpuUserTicks = 0;
        long cpuSystemTicks = 0;
        long cpuOtherTicks = 0;
        long cpuIdleTicks = 0;
        cpuUserTicks    += Long.parseLong(m.group(1 + matchGroupBase));
        cpuOtherTicks   += Long.parseLong(m.group(2 + matchGroupBase));
        cpuSystemTicks  += Long.parseLong(m.group(3 + matchGroupBase));
        cpuIdleTicks    += Long.parseLong(m.group(4 + matchGroupBase));
        cpuOtherTicks   += Long.parseLong(m.group(5 + matchGroupBase));
        cpuSystemTicks  += Long.parseLong(m.group(6 + matchGroupBase));
        cpuSystemTicks  += Long.parseLong(m.group(7 + matchGroupBase));
        CPUTICKS[cpu][0] = cpuUserTicks;
        CPUTICKS[cpu][1] = cpuSystemTicks;
        CPUTICKS[cpu][2] = cpuOtherTicks;
        CPUTICKS[cpu][3] = cpuIdleTicks;
        cpu++;
        line = reader.readLine();
      }
    } catch( IOException _ ) {/**/}
  }

  // --------- /proc/pid/stat --- System and Process cpu ticks -----------
  /** @return number of currently open fds of this process.  */<commit_msg>Fix compiler warnings.<commit_after>package water.util;

import java.io.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Linux /proc file reader.
 *
 * Read tick information for the system and the current process in order to provide
 * stats on the cloud page about CPU utilization.
 *
 * Tick counts are monotonically increasing since boot.
 *
 * Find definitions of /proc file info here.
 * http://man7.org/linux/man-pages/man5/proc.5.html
 */
public abstract class LinuxProcFileReader {
  private static final char[] buffer = new char[8 * 1024]; // MUST be first in file, or else clinit ordering problems
  // Read all of a file, throwing IOE on buffer-full or returning a String
  private static String readFile(File f) throws IOException {
    int bytesRead = 0;
    try(FileReader fr = new FileReader(f)) {
        synchronized(buffer) {
          while (true) {
            int n = fr.read(buffer, bytesRead, buffer.length - bytesRead);
            if( n < 0 ) return new String(buffer, 0, bytesRead);
            bytesRead += n;
            if( bytesRead >= buffer.length ) throw new IOException("LinuxProcFileReader readFile unexpected buffer full");
          }
        }
    }
  }

  // --------- Process ID --------------
  /** @return process id for this node  */
  public static int getProcessID() { return PID; }
  private static final int PID;
  static { PID = getProcessIdFromJMX(); }
  private static int getProcessIdFromJMX() {
    // Note: may fail in some JVM implementations therefore fallback has to be provided.
    // Something like '<pid>@<hostname>', at least in SUN / Oracle JVMs
    final String jvmName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
    final int index = jvmName.indexOf('@');
    return index < 1 ? -1 // part before '@' empty (index = 0) / '@' not found (index = -1)
      : Integer.parseInt(jvmName.substring(0, index));
  }

  // --------- /proc/pid/status --- Number of available processors
  /** @return number of CPUs allowed by this process. */
  public static char getProcessCpusAllowed() { return PROCESS_CPUS_ALLOWED; }
  private static final char PROCESS_CPUS_ALLOWED;
  static { PROCESS_CPUS_ALLOWED = parseProcessStatusFile(readProcessStatusFile(PID)); }
  private static String readProcessStatusFile(int pid) {
    try { return pid== -1 ? null : readFile(new File("/proc/"+pid+"/status")); }
    catch( IOException ignore ) { return null; }
  }
  private static char parseProcessStatusFile(String s) {
    char cpus = (char) Runtime.getRuntime().availableProcessors();
    if(s == null) return cpus;
    try {
      Matcher m = Pattern.compile("Cpus_allowed:\\s+([A-Fa-f0-9,]+)").matcher(s);
      return m.find() ? numSetBitsHex(m.group(1)) : cpus;
    }
    catch( Exception ignore ) { return cpus; }
  }
  /** @return number of set bits in hexadecimal string (chars must be 0-F) */
  private static char numSetBitsHex(String s) {
    // Look-up table for num set bits in 4-bit char
    final int[] bits_set = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
    char nset = 0;
    for(int i = 0; i < s.length(); i++) {
      Character ch = s.charAt(i);
      if (ch == ',') continue;
      int x = Integer.parseInt(ch.toString(), 16);
      nset += bits_set[x];
    }
    return nset;
  }


  // --------- /proc/pid/proc --- process RSS and TotalTicks ----
  /** @return ticks this process was running. */
  public static long getProcessTotalTicks() { return _processTotalTicks; }
  private static long _processTotalTicks = -1;

  /** @return resident set size (RSS) of this process. */
  public static long getProcessRss() { return _processRss; }
  private static long _processRss = -1;

  private static String readProcessProcFile(int pid) {
    try { return pid == -1 ? null : readFile(new File("/proc/"+pid+"/stat")); }
    catch( IOException ignore ) { return null; }
  }
  private static final Pattern procPattern = Pattern.compile(
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + "\\s+" +
    "(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)" + ".*");
  private static void parseProcessProcFile(String s) {
    if (s == null) return;
    try {
      BufferedReader reader = new BufferedReader(new StringReader(s));
      Matcher m = procPattern.matcher(reader.readLine());
      if(!m.matches() ) return;
      long processUserTicks   = Long.parseLong(m.group(14));
      long processSystemTicks = Long.parseLong(m.group(15));
      _processTotalTicks = processUserTicks + processSystemTicks;
      _processRss = Long.parseLong(m.group(24));
    }
    catch (Exception ignore) {}
  }


  // --------- /proc/pid/stat --- System and Process cpu ticks -----------
  /**  Array of ticks:  [cpu number][tick type]
   *
   * tick types are:
   * [0] user ticks
   * [1] system ticks
   * [2] other ticks (i/o)
   * [3] idle ticks
   *
   * @return ticks array for each cpu of the system.
   */
  public static long[][] getCpuTicks() { return CPUTICKS; }
  private static final long[][] CPUTICKS = new long[PROCESS_CPUS_ALLOWED][4/*{user,sys,io,idle}*/];
  /** @return ticks the system was idle.  in general:  idle + busy == 100% */
  public static long getSystemIdleTicks() { return _systemIdleTicks; }
  private static long _systemIdleTicks = -1;
  /** @return ticks the system was up. */
  public static long getSystemTotalTicks() { return _systemTotalTicks; }
  private static long _systemTotalTicks = -1;
  // Read from various /proc files, returning a String or null if not available
  private static String readSystemProcFile() {
    try { return readFile(new File("/proc/stat")); }
    catch( IOException ignore ) { return null; }
  }
  private static final Pattern sysTicksPattern = Pattern.compile("cpu\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+).*");
  private static final Pattern cpuTicksPattern = Pattern.compile("cpu(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+).*");
  /** @param s String containing contents of proc file. */
  private static void parseSystemProcFile(String s) {
    if (s == null) return;
    BufferedReader reader = new BufferedReader(new StringReader(s));
    try { 
      // Read aggregate cpu values
      String line = reader.readLine();
      Matcher m = sysTicksPattern.matcher(line);
      if( !m.matches() ) return;
  
      int matchGroupBase = 0;
      long systemUserTicks = 0;
      long systemSystemTicks = 0;
      long systemOtherTicks = 0;
      systemUserTicks    += Long.parseLong(m.group(1 + matchGroupBase));
      systemOtherTicks   += Long.parseLong(m.group(2 + matchGroupBase));
      systemSystemTicks  += Long.parseLong(m.group(3 + matchGroupBase));
      _systemIdleTicks    = Long.parseLong(m.group(4 + matchGroupBase));
      systemOtherTicks   += Long.parseLong(m.group(5 + matchGroupBase));
      systemSystemTicks  += Long.parseLong(m.group(6 + matchGroupBase));
      systemSystemTicks  += Long.parseLong(m.group(7 + matchGroupBase));
      _systemTotalTicks   = systemUserTicks + systemOtherTicks + systemSystemTicks + _systemIdleTicks;
  
      // Read individual cpu values
      int cpu = 0;
      line = reader.readLine();
      while (line != null) {
        m = cpuTicksPattern.matcher(line);
        if( !m.matches() ) break;
        // Copying algorithm from http://gee.cs.oswego.edu/dl/code/
        // See perfbar.c in gtk_perfbar package.
        // int cpuNum = Integer.parseInt(m.group(1));
        matchGroupBase = 1;
        long cpuUserTicks = 0;
        long cpuSystemTicks = 0;
        long cpuOtherTicks = 0;
        long cpuIdleTicks = 0;
        cpuUserTicks    += Long.parseLong(m.group(1 + matchGroupBase));
        cpuOtherTicks   += Long.parseLong(m.group(2 + matchGroupBase));
        cpuSystemTicks  += Long.parseLong(m.group(3 + matchGroupBase));
        cpuIdleTicks    += Long.parseLong(m.group(4 + matchGroupBase));
        cpuOtherTicks   += Long.parseLong(m.group(5 + matchGroupBase));
        cpuSystemTicks  += Long.parseLong(m.group(6 + matchGroupBase));
        cpuSystemTicks  += Long.parseLong(m.group(7 + matchGroupBase));
        CPUTICKS[cpu][0] = cpuUserTicks;
        CPUTICKS[cpu][1] = cpuSystemTicks;
        CPUTICKS[cpu][2] = cpuOtherTicks;
        CPUTICKS[cpu][3] = cpuIdleTicks;
        cpu++;
        line = reader.readLine();
      }
    } catch( IOException ignore ) {/**/}
  }

  // --------- /proc/pid/stat --- System and Process cpu ticks -----------
  /** @return number of currently open fds of this process.  */
{'fraction_non_alphanumeric': 0.16395422393467135, 'fraction_numerical': 0.010351371556731267, 'mean_word_length': 2.7591872027669693, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3747606', 'n_tokens_mistral': 5781, 'n_tokens_neox': 5626, 'n_words': 1472}