<commit_before>#include "data.table.h"
#include <errno.h>
#include <unistd.h>  // for access()
#include <fcntl.h>
#include <time.h>
#ifdef WIN32
#include <sys/types.h>
#include <sys/stat.h>
#include <io.h>
#define WRITE _write
#define CLOSE _close
#else
#define WRITE write
#define CLOSE close
#endif

static inline int maxStrLen(SEXP x, int na_len) {
  // The max nchar of any string in a column or factor level
  int max=na_len, nrow=length(x);
  for (int i=0; i<nrow; i++) {
    int l = LENGTH(STRING_ELT(x,i));
    if (l>max) max=l;
  }
  // TODO if(quote) count them here (we hopped already to get LENGTH), add that quote count to max
  //      exactly and remove the *(1+quote) later
  //      if there are no quotes present in any string in a column save in quotePresent[ncol] lookup
  //      and switch to memcpy when it's known no quotes are present to be escaped for that column
  return max;
}

#define QUOTE_FIELD \
  *ch++ = QUOTE; \
  for (const char *ch2 = CHAR(str); *ch2 != '\0'; ch2++) { \
    if (*ch2 == QUOTE) *ch++ = ESCAPE_QUOTE; \
    if (qmethod_escape && *ch2 == ESCAPE) *ch++ = ESCAPE; \
    *ch++ = *ch2; \
  } \
  *ch++ = QUOTE

#define NUM_SF 15
#define DECIMAL_SEP '.'  // TODO allow other decimal separator e.g. ','

// Globals for this file only (written once to hold parameters passed from R level)
static int na_len;
static const char *na_str;

static inline void writeInteger(int x, char **thisCh)
{
  char *ch = *thisCh;
  if (x == NA_INTEGER) {
    if (na_len) { memcpy(ch, na_str, na_len); ch += na_len; }
  } else if (x == 0) {
    *ch++ = '0';
  } else {
    if (x<0) { *ch++ = '-'; x=-x; }
    // avoid log() call for speed. write backwards then reverse when we know how long
    int width = 0;
    while (x>0) { *ch++ = '0'+x%10; x /= 10; width++; }
    for (int i=width/2; i>0; i--) {
      char tmp=*(ch-i);
      *(ch-i) = *(ch-width+i-1);
      *(ch-width+i-1) = tmp;
    }
  }
  *thisCh = ch;
}
/*
typedef union {
  double d;
  struct {
    unsigned long long fraction : 52;
    unsigned int       exponent : 11;
    unsigned int       signbit  :  1;
  } parts;
} double_cast;
*/

union {
  double d;
  unsigned long long ull;
} u;


static inline void writeNumeric(double x, char **thisCh)
{
  // hand-rolled / specialized for speed
  // *thisCh is safely the output destination with enough space (ensured via calculating maxLineLen up front)
  // technique similar to base R (format.c:formatReal and printutils.c:EncodeReal0)
  // differences/tricks :
  //   i) no buffers. writes straight to the final file buffer passed to write()
  //  ii) no C libary calls such as sprintf() where the fmt string has to be interpretted over and over
  // iii) no need to return variables or flags.  Just writes.
  //  iv) shorter, easier to read and reason with. In one self contained place.
  char *ch = *thisCh;
  if (!R_FINITE(x)) {
    if (ISNAN(x)) {
      if (na_len) { memcpy(ch, na_str, na_len); ch += na_len; }  // by default na_len==0 and the memcpy call will be skipped
    } else if (x>0) {
      *ch++ = 'I'; *ch++ = 'n'; *ch++ = 'f';
    } else {
      *ch++ = '-'; *ch++ = 'I'; *ch++ = 'n'; *ch++ = 'f';
    }
  } else if (x == 0.0) {
    *ch++ = '0';   // and we're done.  so much easier rather than passing back special cases
  } else {
    if (x < 0.0) { *ch++ = '-'; x = -x; }  // and we're done on sign, already written. no need to pass back sign
    
    /*
    int exp = (int)floorl(log10l((long double)x));
    unsigned long long l = (unsigned long long)((long double)x * powl(10.0L, NUM_SF-exp));
    */
    
    u.d = x;
    unsigned long long fraction = u.ull & ((1ULL<<52)-1);
    unsigned int exponent = (u.ull>>52) & 0x7FF;
    
    /*
    double_cast d;
    d.d = x;
    */
    long double y = 1.0 + fraction / 4503599627370496.0L;  // 2^52 as long double
    //e2 = 0;
    //int exp=0;
    int e2 = exponent-1023;
    //long double y = (long double)frexpl(x, &e2);
    // int nd = (int)(e2 * 0.30102999566);  // * log10(2)
    
    // y *= (powl(2.0, e2) / powl(10.0, nd));  // TODO: easier/faster more accurate way?
    // TODO: don't need this since y was 1.* above so surely we know
    
    y = ldexpl(y, e2);
   
    int exp = (int)floorl(log10l(y));
    unsigned long long l = (unsigned long long)(y * powl(10.0L, NUM_SF-exp));
    /*
    if (y<1) { y *= 1e15; exp++; }
    else  
    y *= 1e15;
    unsigned long long l = (unsigned long long)y;
    int exp = 
    
    int m = log10(2^e2)
    y * powl(10.0, NUM_DF)
    
    int exp = (int)floor(log10(x));
    
    unsigned long long l = (unsigned long long)(y *
                                                 (long double)powl((long double)10,(long double)(NUM_SF-exp)));
    //exp+=nd;
    */
    
    // TODO?: use lookup table like base R? ................. ^^^^<commit_msg>Will revisit the two fwrite tests on windows: 1729.3 and 1729.9<commit_after>#include "data.table.h"
#include <errno.h>
#include <unistd.h>  // for access()
#include <fcntl.h>
#include <time.h>
#ifdef WIN32
#include <sys/types.h>
#include <sys/stat.h>
#include <io.h>
#define WRITE _write
#define CLOSE _close
#else
#define WRITE write
#define CLOSE close
#endif

static inline int maxStrLen(SEXP x, int na_len) {
  // The max nchar of any string in a column or factor level
  int max=na_len, nrow=length(x);
  for (int i=0; i<nrow; i++) {
    int l = LENGTH(STRING_ELT(x,i));
    if (l>max) max=l;
  }
  // TODO if(quote) count them here (we hopped already to get LENGTH), add that quote count to max
  //      exactly and remove the *(1+quote) later
  //      if there are no quotes present in any string in a column save in quotePresent[ncol] lookup
  //      and switch to memcpy when it's known no quotes are present to be escaped for that column
  return max;
}

#define QUOTE_FIELD \
  *ch++ = QUOTE; \
  for (const char *ch2 = CHAR(str); *ch2 != '\0'; ch2++) { \
    if (*ch2 == QUOTE) *ch++ = ESCAPE_QUOTE; \
    if (qmethod_escape && *ch2 == ESCAPE) *ch++ = ESCAPE; \
    *ch++ = *ch2; \
  } \
  *ch++ = QUOTE

#define NUM_SF 15
#define DECIMAL_SEP '.'  // TODO allow other decimal separator e.g. ','

// Globals for this file only (written once to hold parameters passed from R level)
static int na_len;
static const char *na_str;

static inline void writeInteger(int x, char **thisCh)
{
  char *ch = *thisCh;
  if (x == NA_INTEGER) {
    if (na_len) { memcpy(ch, na_str, na_len); ch += na_len; }
  } else if (x == 0) {
    *ch++ = '0';
  } else {
    if (x<0) { *ch++ = '-'; x=-x; }
    // avoid log() call for speed. write backwards then reverse when we know how long
    int width = 0;
    while (x>0) { *ch++ = '0'+x%10; x /= 10; width++; }
    for (int i=width/2; i>0; i--) {
      char tmp=*(ch-i);
      *(ch-i) = *(ch-width+i-1);
      *(ch-width+i-1) = tmp;
    }
  }
  *thisCh = ch;
}

union {
  double d;
  unsigned long long ull;
} u;

static inline void writeNumeric(double x, char **thisCh)
{
  // hand-rolled / specialized for speed
  // *thisCh is safely the output destination with enough space (ensured via calculating maxLineLen up front)
  // technique similar to base R (format.c:formatReal and printutils.c:EncodeReal0)
  // differences/tricks :
  //   i) no buffers. writes straight to the final file buffer passed to write()
  //  ii) no C libary calls such as sprintf() where the fmt string has to be interpretted over and over
  // iii) no need to return variables or flags.  Just writes.
  //  iv) shorter, easier to read and reason with. In one self contained place.
  char *ch = *thisCh;
  if (!R_FINITE(x)) {
    if (ISNAN(x)) {
      if (na_len) { memcpy(ch, na_str, na_len); ch += na_len; }  // by default na_len==0 and the memcpy call will be skipped
    } else if (x>0) {
      *ch++ = 'I'; *ch++ = 'n'; *ch++ = 'f';
    } else {
      *ch++ = '-'; *ch++ = 'I'; *ch++ = 'n'; *ch++ = 'f';
    }
  } else if (x == 0.0) {
    *ch++ = '0';   // and we're done.  so much easier rather than passing back special cases
  } else {
    if (x < 0.0) { *ch++ = '-'; x = -x; }  // and we're done on sign, already written. no need to pass back sign
    
    /*
    int exp = (int)floorl(log10l((long double)x));
    unsigned long long l = (unsigned long long)((long double)x * powl(10.0L, NUM_SF-exp));
    */
    
    u.d = x;
    unsigned long long fraction = u.ull & ((1ULL<<52)-1);
    unsigned int exponent = (u.ull>>52) & 0x7FF;
    long double y = 1.0 + fraction / 4503599627370496.0L;  // 2^52 as long double
    y = ldexpl(y, exponent-1023);
    // long double y = (long double)frexpl(x, &e2);
    // int nd = (int)(e2 * 0.30102999566);  // * log10(2)
    // y *= (powl(2.0, e2) / powl(10.0, nd));  // doesn't seem efficient or accurate
    // since y was 1.* above so surely we know

    int exp = (int)floorl(log10l(y));
    unsigned long long l = (unsigned long long)(y * powl(10.0L, NUM_SF-exp));
    /*
    if (y<1) { y *= 1e15; exp++; }
    else  
    y *= 1e15;
    unsigned long long l = (unsigned long long)y;
    int exp = 
    
    int m = log10(2^e2)
    y * powl(10.0, NUM_DF)
    
    int exp = (int)floor(log10(x));
    
    unsigned long long l = (unsigned long long)(y *
                                                 (long double)powl((long double)10,(long double)(NUM_SF-exp)));
    //exp+=nd;
    */
    
    // TODO?: use lookup table like base R? ................. ^^^^
{'fraction_non_alphanumeric': 0.15908360989187453, 'fraction_numerical': 0.028048388823466438, 'mean_word_length': 2.467706013363029, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1245714', 'n_tokens_mistral': 3585, 'n_tokens_neox': 3418, 'n_words': 1157}