<commit_before>#include "data.table.h"
#ifdef _OPENMP
#include <pthread.h>
#endif
#include <errno.h>     // errno
#include <ctype.h>     // isspace

static int  DTthreads = -1;           // Never read directly hence static; use getDTthreads(). -1 so we know for sure initDTthreads() ran and set it >= 1.
static bool RestoreAfterFork = true;  // see #2885 in v1.12.0

static int getIntEnv(const char *name, int def)
{
  const char *val = getenv(name);
  if (val==NULL) return def;
  size_t nchar = strlen(val);
  if (nchar==0) return def;
  char *end;
  errno = 0;
  long int ans = strtol(val, &end, 10);  // ignores leading whitespace. If it fully consumed the string, *end=='\0' and isspace('\0')==false
  while (isspace(*end)) end++;  // ignore trailing whitespace
  if (errno || (size_t)(end-val)!=nchar || ans<1 || ans>INT_MAX) {
    warning("Ignoring invalid %s==\"%s\". Not an integer >= 1. Please remove any characters that are not a digit [0-9]. See ?data.table::setDTthreads.", name, val);
    return def;
  }
  return (int)ans;
}

static inline int imin(int a, int b) { return a < b ? a : b; }
static inline int imax(int a, int b) { return a > b ? a : b; }

void initDTthreads() {
  // called at package startup from init.c
  // also called by setDTthreads(threads=NULL) (default) to reread environment variables; see setDTthreads below
  // No verbosity here in this setter. Verbosity is in getDTthreads(verbose=TRUE)
#ifndef _OPENMP
  DTthreads = 1;
#else
  int ans = omp_get_num_procs();  // starting point is all logical CPUs. This is a hard limit; user cannot achieve more than this.
  if (ans<=1) {
    // covers <=0 too in case omp_get_num_procs() returns <= 0 (perhaps error, or unsupported)
    ans = 1;  // # nocov
  } else {
    int perc = getIntEnv("R_DATATABLE_NUM_PROCS_PERCENT", 50); // use "NUM_PROCS" to use the same name as the OpenMP function this uses
    // 50% of logical CPUs by default; half of 8 is 4 on laptop with 4 cores. Leaves plenty of room for other processes: #3395 & #3298
    if (perc<=1 || perc>100) {
      warning("Ignoring invalid R_DATATABLE_NUM_PROCS_PERCENT==%d. If used it must be an integer between 2 and 100. Default is 50. See ?setDTtheads.", perc);
      // not allowing 1 is to catch attempts to use 1 or 1.0 to represent 100%.
      perc = 50;
    }
    ans = imax(ans*perc/100, 1);
    ans = imin(ans, omp_get_thread_limit());  // honors OMP_THREAD_LIMIT when OpenMP started; e.g. CRAN sets this to 2. Often INT_MAX meaning unlimited/unset
    ans = imin(ans, omp_get_max_threads());   // honors OMP_NUM_THREADS when OpenMP started, plus reflects any omp_set_* calls made since
    ans = imax(ans, 1);                       // just in case omp_get_* returned <= 0 for any reason
    // max_threads() -vs- num_procs(): https://software.intel.com/en-us/forums/intel-visual-fortran-compiler-for-windows/topic/302866
    ans = imin(ans, getIntEnv("R_DATATABLE_NUM_THREADS", INT_MAX));
    ans = imin(ans, getIntEnv("OMP_THREAD_LIMIT", INT_MAX));  // user might expect `Sys.setenv(OMP_THREAD_LIMIT=2);setDTthreads()` to work. Satisfy this
    ans = imin(ans, getIntEnv("OMP_NUM_THREADS", INT_MAX));   //   expectation by reading them again now. OpenMP just reads them on startup (quite reasonably)
  }
  DTthreads = ans;
#endif
}

int getDTthreads() {
  // this is the main getter used by all parallel regions; they specify num_threads(getDTthreads())
  // Therefore keep it light, simple and robust. Local static variable. initDTthreads() ensures 1 <= DTthreads <= omp_get_num_proc()
  return DTthreads;
}

static const char *mygetenv(const char *name) {
  const char *ans = getenv(name);
  if (ans==NULL) ans="";
  return ans;
}

SEXP getDTthreads_R(SEXP verbose) {
  if (!isLogical(verbose) || LENGTH(verbose)!=1 || INTEGER(verbose)[0]==NA_LOGICAL) error("'verbose' must be TRUE or FALSE");
  if (LOGICAL(verbose)[0]) {
    #ifndef _OPENMP
      Rprintf("This installation of data.table has not been compiled with OpenMP support.\n");
    #else
      // this output is captured, paste0(collapse="; ")'d, and placed at the end of test.data.table() for display in the last 13 lines of CRAN check logs
      Rprintf("omp_get_num_procs()==%d\n", omp_get_num_procs());
      const char *p = mygetenv("R_DATATABLE_NUM_PROCS_PERCENT");
      Rprintf("R_DATATABLE_NUM_PROCS_PERCENT==\"%s\" %s\n", p, p[0]=='\0' ? "(default 50)" : "");
      Rprintf("R_DATATABLE_NUM_THREADS==\"%s\"\n", mygetenv("R_DATATABLE_NUM_THREADS"));
      Rprintf("omp_get_thread_limit()==%d\n", omp_get_thread_limit());
      Rprintf("omp_get_max_threads()==%d\n", omp_get_max_threads());
      Rprintf("OMP_THREAD_LIMIT==\"%s\"\n", mygetenv("OMP_THREAD_LIMIT"));  // CRAN sets to 2
      Rprintf("OMP_NUM_THREADS==\"%s\"\n",  mygetenv("OMP_NUM_THREADS"));
      Rprintf("data.table is using %d threads. This is set on startup, and by setDTthreads(). See ?setDTthreads.\n", getDTthreads());
      Rprintf("RestoreAfterFork==%s\n", RestoreAfterFork ? "true" : "false");
    #endif
  }
  return ScalarInteger(getDTthreads());
}

SEXP setDTthreads(SEXP threads, SEXP restore_after_fork, SEXP percent) {
  if (!isNull(restore_after_fork)) {
    if (!isLogical(restore_after_fork) || LOGICAL(restore_after_fork)[0]==NA_LOGICAL) {
      error("restore_after_fork= must be TRUE, FALSE, or NULL (default). getDTthreads(verbose=TRUE) reports the current setting.\n");
    }
    RestoreAfterFork = LOGICAL(restore_after_fork)[0];  // # nocov
  }
  int old = DTthreads;
  if (isNull(threads)) {
    initDTthreads();
    // Rerun exactly the same function used on startup (re-reads env variables); this is now default setDTthreads() behavior from 1.12.2
    // Allows robust testing of environment variables using Sys.setenv() to experiment.
    // Default  is now (as from 1.12.2) threads=NULL which re-reads environment variables.
    // If a CPU has been unplugged (high end servers allow live hardware replacement) then omp_get_num_procs() will
    // reflect that and a call to setDTthreads(threads=NULL) will update DTthreads. 
  } else {
    int n=0, protecti=0;
    if (length(threads)!=1) error("threads= must be either NULL (default) or a single number. It has length %d", length(threads));
    if (isReal(threads)) { threads = PROTECT(coerceVector(threads, INTSXP)); protecti++; }
    if (!isInteger(threads)) error("threads= must be either NULL (default) or type integer/numeric");
    if ((n=INTEGER(threads)[0]) < 0) {  // <0 catches NA too since NA is negative (INT_MIN)
      error("threads= must be either NULL or a single integer >= 0. See ?setDTthreads.");
    }
    UNPROTECT(protecti);
    int num_procs = imax(omp_get_num_procs(), 1); // max just in case omp_get_num_procs() returns <= 0 (perhaps error, or unsupported)
    if (!isLogical(percent) || length(percent)!=1 || LOGICAL(percent)[0]==NA_LOGICAL) {<commit_msg>pass 3 tests (1895, 1997.09 and 1997.11) when OpenMP is missing; #3435<commit_after>#include "data.table.h"
#ifdef _OPENMP
#include <pthread.h>
#endif
#include <errno.h>     // errno
#include <ctype.h>     // isspace

static int  DTthreads = -1;           // Never read directly hence static; use getDTthreads(). -1 so we know for sure initDTthreads() ran and set it >= 1.
static bool RestoreAfterFork = true;  // see #2885 in v1.12.0

static int getIntEnv(const char *name, int def)
{
  const char *val = getenv(name);
  if (val==NULL) return def;
  size_t nchar = strlen(val);
  if (nchar==0) return def;
  char *end;
  errno = 0;
  long int ans = strtol(val, &end, 10);  // ignores leading whitespace. If it fully consumed the string, *end=='\0' and isspace('\0')==false
  while (isspace(*end)) end++;  // ignore trailing whitespace
  if (errno || (size_t)(end-val)!=nchar || ans<1 || ans>INT_MAX) {
    warning("Ignoring invalid %s==\"%s\". Not an integer >= 1. Please remove any characters that are not a digit [0-9]. See ?data.table::setDTthreads.", name, val);
    return def;
  }
  return (int)ans;
}

static inline int imin(int a, int b) { return a < b ? a : b; }
static inline int imax(int a, int b) { return a > b ? a : b; }

void initDTthreads() {
  // called at package startup from init.c
  // also called by setDTthreads(threads=NULL) (default) to reread environment variables; see setDTthreads below
  // No verbosity here in this setter. Verbosity is in getDTthreads(verbose=TRUE)
  int ans = omp_get_num_procs();  // starting point is all logical CPUs. This is a hard limit; user cannot achieve more than this.
                                  // ifndef _OPENMP then myomp.h defines this to be 1
  int perc = getIntEnv("R_DATATABLE_NUM_PROCS_PERCENT", 50); // use "NUM_PROCS" to use the same name as the OpenMP function this uses
  // 50% of logical CPUs by default; half of 8 is 4 on laptop with 4 cores. Leaves plenty of room for other processes: #3395 & #3298
  if (perc<=1 || perc>100) {
    warning("Ignoring invalid R_DATATABLE_NUM_PROCS_PERCENT==%d. If used it must be an integer between 2 and 100. Default is 50. See ?setDTtheads.", perc);
    // not allowing 1 is to catch attempts to use 1 or 1.0 to represent 100%.
    perc = 50;
  }
  ans = imax(ans*perc/100, 1);
  ans = imin(ans, omp_get_thread_limit());  // honors OMP_THREAD_LIMIT when OpenMP started; e.g. CRAN sets this to 2. Often INT_MAX meaning unlimited/unset
  ans = imin(ans, omp_get_max_threads());   // honors OMP_NUM_THREADS when OpenMP started, plus reflects any omp_set_* calls made since
  ans = imax(ans, 1);                       // just in case omp_get_* returned <= 0 for any reason
  // max_threads() -vs- num_procs(): https://software.intel.com/en-us/forums/intel-visual-fortran-compiler-for-windows/topic/302866
  ans = imin(ans, getIntEnv("R_DATATABLE_NUM_THREADS", INT_MAX));
  ans = imin(ans, getIntEnv("OMP_THREAD_LIMIT", INT_MAX));  // user might expect `Sys.setenv(OMP_THREAD_LIMIT=2);setDTthreads()` to work. Satisfy this
  ans = imin(ans, getIntEnv("OMP_NUM_THREADS", INT_MAX));   //   expectation by reading them again now. OpenMP just reads them on startup (quite reasonably)
  DTthreads = ans;
}

int getDTthreads() {
  // this is the main getter used by all parallel regions; they specify num_threads(getDTthreads())
  // Therefore keep it light, simple and robust. Local static variable. initDTthreads() ensures 1 <= DTthreads <= omp_get_num_proc()
  return DTthreads;
}

static const char *mygetenv(const char *name) {
  const char *ans = getenv(name);
  if (ans==NULL) ans="";
  return ans;
}

SEXP getDTthreads_R(SEXP verbose) {
  if (!isLogical(verbose) || LENGTH(verbose)!=1 || INTEGER(verbose)[0]==NA_LOGICAL) error("'verbose' must be TRUE or FALSE");
  if (LOGICAL(verbose)[0]) {
    #ifndef _OPENMP
      Rprintf("This installation of data.table has not been compiled with OpenMP support.\n");
    #endif
    // this output is captured, paste0(collapse="; ")'d, and placed at the end of test.data.table() for display in the last 13 lines of CRAN check logs
    Rprintf("omp_get_num_procs()==%d\n", omp_get_num_procs());
    const char *p = mygetenv("R_DATATABLE_NUM_PROCS_PERCENT");
    Rprintf("R_DATATABLE_NUM_PROCS_PERCENT==\"%s\" %s\n", p, p[0]=='\0' ? "(default 50)" : "");
    Rprintf("R_DATATABLE_NUM_THREADS==\"%s\"\n", mygetenv("R_DATATABLE_NUM_THREADS"));
    Rprintf("omp_get_thread_limit()==%d\n", omp_get_thread_limit());
    Rprintf("omp_get_max_threads()==%d\n", omp_get_max_threads());
    Rprintf("OMP_THREAD_LIMIT==\"%s\"\n", mygetenv("OMP_THREAD_LIMIT"));  // CRAN sets to 2
    Rprintf("OMP_NUM_THREADS==\"%s\"\n",  mygetenv("OMP_NUM_THREADS"));
    Rprintf("data.table is using %d threads. This is set on startup, and by setDTthreads(). See ?setDTthreads.\n", getDTthreads());
    Rprintf("RestoreAfterFork==%s\n", RestoreAfterFork ? "true" : "false");
  }
  return ScalarInteger(getDTthreads());
}

SEXP setDTthreads(SEXP threads, SEXP restore_after_fork, SEXP percent) {
  if (!isNull(restore_after_fork)) {
    if (!isLogical(restore_after_fork) || LOGICAL(restore_after_fork)[0]==NA_LOGICAL) {
      error("restore_after_fork= must be TRUE, FALSE, or NULL (default). getDTthreads(verbose=TRUE) reports the current setting.\n");
    }
    RestoreAfterFork = LOGICAL(restore_after_fork)[0];  // # nocov
  }
  int old = DTthreads;
  if (isNull(threads)) {
    initDTthreads();
    // Rerun exactly the same function used on startup (re-reads env variables); this is now default setDTthreads() behavior from 1.12.2
    // Allows robust testing of environment variables using Sys.setenv() to experiment.
    // Default  is now (as from 1.12.2) threads=NULL which re-reads environment variables.
    // If a CPU has been unplugged (high end servers allow live hardware replacement) then omp_get_num_procs() will
    // reflect that and a call to setDTthreads(threads=NULL) will update DTthreads. 
  } else {
    int n=0, protecti=0;
    if (length(threads)!=1) error("threads= must be either NULL (default) or a single number. It has length %d", length(threads));
    if (isReal(threads)) { threads = PROTECT(coerceVector(threads, INTSXP)); protecti++; }
    if (!isInteger(threads)) error("threads= must be either NULL (default) or type integer/numeric");
    if ((n=INTEGER(threads)[0]) < 0) {  // <0 catches NA too since NA is negative (INT_MIN)
      error("threads= must be either NULL or a single integer >= 0. See ?setDTthreads.");
    }
    UNPROTECT(protecti);
    int num_procs = imax(omp_get_num_procs(), 1); // max just in case omp_get_num_procs() returns <= 0 (perhaps error, or unsupported)
    if (!isLogical(percent) || length(percent)!=1 || LOGICAL(percent)[0]==NA_LOGICAL) {
{'fraction_non_alphanumeric': 0.13243084167157151, 'fraction_numerical': 0.016995291347851677, 'mean_word_length': 3.916094032549729, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6230135', 'n_tokens_mistral': 4730, 'n_tokens_neox': 4522, 'n_words': 1443}