<commit_before>var sys = require('sys');
var constants = require('./constants');

function writer()
{
   this.data = "";
}

writer.prototype.zstring = function(s)
{
   this.data += s + "\u0000";
   return this;
}

//
//  length-coded number
//
//  Value Of     # Of Bytes  Description
//  First Byte   Following
//  ----------   ----------- -----------
//  0-250        0           = value of first byte
//  251          0           column value = NULL
//                           only appropriate in a Row Data Packet
//  252          2           = value of following 16-bit word
//  253          3           = value of following 24-bit word
//  254          8           = value of following 64-bit word
//
writer.prototype.lcnum = function(n)
{
   if (n < 251)
       this.data += String.fromCharCode(n);
   else if (n < 0xffff)
   {
       this.data += String.fromCharCode(252);
       this.data += String.fromCharCode( n & 0xff );
       this.data += String.fromCharCode( (n >> 8) & 0xff );
   } else if (n < 0xffffff)
   {
       this.data += String.fromCharCode(253);
       this.data += String.fromCharCode( n & 0xff );
       this.data += String.fromCharCode( (n >> 8) & 0xff );
       this.data += String.fromCharCode( (n >> 16) & 0xff );
   } 
   /*
      TODO: 64 bit number
   */
   return this;
}

//
// write length-coded string to the buffer
//
writer.prototype.lcstring = function(s)
{
   this.lcnum(s.length);
   this.data += s;
   return this;
}

writer.prototype.add = function(s)
{
   if (typeof s == "string")      // add string bufer
       this.data += s; 
   else if (typeof s == "number") // add four byte integer
   {
       this.data += String.fromCharCode( s & 0xff );
       this.data += String.fromCharCode( (s >> 8)  & 0xff );
       this.data += String.fromCharCode( (s >> 16) & 0xff );
       this.data += String.fromCharCode( (s >> 24) & 0xff );
   }
   return this;
}

writer.prototype.int2 = function(s)
{
    this.data += String.fromCharCode( s & 0xff );
    this.data += String.fromCharCode( (s >> 8)  & 0xff );
}

writer.prototype.addHeader = function(n)
{
    var length = this.data.length;
    var header = "";
    header += String.fromCharCode( length     & 0xff );
    header += String.fromCharCode( length>>8  & 0xff );
    header += String.fromCharCode( length>>16 & 0xff );
    var packet_num = n ? n : 0;
    header += String.fromCharCode( packet_num );
    this.data = header + this.data;
    return this;
}

function reader(data)
{
   this.data = data;
   this.pos = 0;

}

// deserialise mysql binary field
reader.prototype.unpackBinary = function(type, unsigned)
{
    // debug dump
    //return "_not_implemented_ " + constants.type_names[type] + " " + sys.inspect(this.data);

    var result;
    switch (type)
    {
    case constants.types.MYSQL_TYPE_STRING:
    case constants.types.MYSQL_TYPE_VAR_STRING:
    case constants.types.MYSQL_TYPE_BLOB:
        result = this.lcstring();
        break;
    case constants.types.MYSQL_TYPE_LONG:
        result = this.num(4);
        break;
    case constants.types.MYSQL_TYPE_LONGLONG:
        result = this.num(8);
        break;
    case constants.types.MYSQL_TYPE_NEWDECIMAL:
        result = parseFloat(this.lcstring());
        break;
    default:
        result = "_not_implemented_ " + constants.type_names[type] + " " + sys.inspect(this.data); //todo: throw exception here
    }
    return result;
}

// read n-bytes number 
reader.prototype.num = function(numbytes)
{
    var res = 0;
    var factor = 1;
    for (var i=0; i < numbytes; ++i)
    {
        res += this.data.charCodeAt(this.pos) * factor;<commit_msg>MYSQL_TYPE_DATE, MYSQL_TYPE_TIME, MYSQL_TYPE_DATETIME, MYSQL_TYPE_TIMESTAMP binary packets support<commit_after>var sys = require('sys');
var constants = require('./constants');

function writer()
{
   this.data = "";
}

writer.prototype.zstring = function(s)
{
   this.data += s + "\u0000";
   return this;
}

//
//  length-coded number
//
//  Value Of     # Of Bytes  Description
//  First Byte   Following
//  ----------   ----------- -----------
//  0-250        0           = value of first byte
//  251          0           column value = NULL
//                           only appropriate in a Row Data Packet
//  252          2           = value of following 16-bit word
//  253          3           = value of following 24-bit word
//  254          8           = value of following 64-bit word
//
writer.prototype.lcnum = function(n)
{
   if (n < 251)
       this.data += String.fromCharCode(n);
   else if (n < 0xffff)
   {
       this.data += String.fromCharCode(252);
       this.data += String.fromCharCode( n & 0xff );
       this.data += String.fromCharCode( (n >> 8) & 0xff );
   } else if (n < 0xffffff)
   {
       this.data += String.fromCharCode(253);
       this.data += String.fromCharCode( n & 0xff );
       this.data += String.fromCharCode( (n >> 8) & 0xff );
       this.data += String.fromCharCode( (n >> 16) & 0xff );
   } 
   /*
      TODO: 64 bit number
   */
   return this;
}

//
// write length-coded string to the buffer
//
writer.prototype.lcstring = function(s)
{
   this.lcnum(s.length);
   this.data += s;
   return this;
}

writer.prototype.add = function(s)
{
   if (typeof s == "string")      // add string bufer
       this.data += s; 
   else if (typeof s == "number") // add four byte integer
   {
       this.data += String.fromCharCode( s & 0xff );
       this.data += String.fromCharCode( (s >> 8)  & 0xff );
       this.data += String.fromCharCode( (s >> 16) & 0xff );
       this.data += String.fromCharCode( (s >> 24) & 0xff );
   }
   return this;
}

writer.prototype.int2 = function(s)
{
    this.data += String.fromCharCode( s & 0xff );
    this.data += String.fromCharCode( (s >> 8)  & 0xff );
}

writer.prototype.addHeader = function(n)
{
    var length = this.data.length;
    var header = "";
    header += String.fromCharCode( length     & 0xff );
    header += String.fromCharCode( length>>8  & 0xff );
    header += String.fromCharCode( length>>16 & 0xff );
    var packet_num = n ? n : 0;
    header += String.fromCharCode( packet_num );
    this.data = header + this.data;
    return this;
}

function reader(data)
{
   this.data = data;
   this.pos = 0;

}

// libmysql sets all fields to zero when binary packet has zero length
function zeroTime()
{
   // todo: check how zero date is serialized to output in mysql
   return new Date(); 
}

reader.prototype.unpackBinaryTime = function()
{
    var length = this.lcnum();
    if (length == 0)
       return zeroTime();
    var sign = this.num(1);
    var day = this.num(4);
    var hour = this.num(1);
    var minutes = this.num(1);
    var seconds = this.num(1);
    var millisec = (length > 8) ? this.num(4) : 0;
    if (day != 0)
        hour += day*24;
    var millisec_time_val = millisec + seconds*1000 + minutes*60000 + hour*3600000;
    if (sign != 0)
        millisec_time_val *= -1;
    return millisec_time_val;
}

reader.prototype.unpackBinaryDateTime = function()
{
    var length = this.lcnum();
    if (length == 0)
       return zeroTime();

    var y = this.num(2);
    var m = this.num(1);
    var d = this.num(1);

    var hour = 0;
    var min = 0;
    var sec = 0;
    if (length > 4)
    {
       hour = this.num(1);
       min = this.num(1);
       sec = this.num(1);
    }
    var millisec = (length > 8) ? this.num(4) : 0;
    var dt = new Date();
    dt.setYear(y);
    dt.setMonth(m); 
    dt.setDate(d);
    dt.setHours(hour);
    dt.setMinutes(min);
    dt.setSeconds(sec);
    dt.setMilliseconds(millisec);
    return dt;
}

reader.prototype.unpackBinaryDate = function()
{
    var length = this.lcnum();
    if (length == 0)
       return zeroTime();

    var y = this.num(2);
    var m = this.num(1);
    var d = this.num(1);
    var dt = new Date();
    dt.setYear(y);
    dt.setMonth(m); 
    dt.setDate(d);

    return dt;
}

// deserialise mysql binary field
reader.prototype.unpackBinary = function(type, unsigned)
{
    // debug dump
    //return "_not_implemented_ " + constants.type_names[type] + " " + sys.inspect(this.data);

    var result;
    switch (type)
    {
    case constants.types.MYSQL_TYPE_STRING:
    case constants.types.MYSQL_TYPE_VAR_STRING:
    case constants.types.MYSQL_TYPE_BLOB:
        result = this.lcstring();
        break;
    case constants.types.MYSQL_TYPE_LONG:
        result = this.num(4);
        break;
    case constants.types.MYSQL_TYPE_LONGLONG:
        result = this.num(8);
        break;
    case constants.types.MYSQL_TYPE_NEWDECIMAL:
        result = parseFloat(this.lcstring());
        break;
/*
  MYSQL_TYPE_TIMESTAMP: 7,
  MYSQL_TYPE_LONGLONG: 8,
  MYSQL_TYPE_INT24: 9,
  MYSQL_TYPE_DATE: 10,
  MYSQL_TYPE_TIME: 11,
  MYSQL_TYPE_DATETIME: 12,
  MYSQL_TYPE_YEAR: 13,
  MYSQL_TYPE_NEWDATE: 14,

*/
    case constants.types.MYSQL_TYPE_DATE:
         return this.unpackBinaryDate();
    case constants.types.MYSQL_TYPE_TIME:
         return this.unpackBinaryTime();
    case constants.types.MYSQL_TYPE_DATETIME:
    case constants.types.MYSQL_TYPE_TYMESTAMP:
         return this.unpackBinaryDateTime();
    default:
        result = "_not_implemented_ " + constants.type_names[type] + " " + sys.inspect(this.data); //todo: throw exception here
    }
    return result;
}

// read n-bytes number 
reader.prototype.num = function(numbytes)
{
    var res = 0;
    var factor = 1;
    for (var i=0; i < numbytes; ++i)
    {
        res += this.data.charCodeAt(this.pos) * factor;
{'fraction_non_alphanumeric': 0.14257133831736338, 'fraction_numerical': 0.023270506475729178, 'mean_word_length': 2.251626155426224, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3265594', 'n_tokens_mistral': 3361, 'n_tokens_neox': 3217, 'n_words': 772}