<commit_before>// YAML.node, © 2011 <NAME>
// MIT-licensed. (See the included LICENSE file.)

var fs = require('fs'),
    binding = exports.capi = require('./build/Release/binding');


// Most of these were derived from: http://yaml.org/type/
// Also borrows from tenderlove's `Psych::ScalarScanner`. (MIT-licensed)
var definitelyNonNumericRe = /^[a-z~]/i,
    definitelyNonBooleanRe = /^[^ytonf~]/i,
    canParseIntRe   = /^[-+]?(0x[0-9a-fA-F_]+|[\d_]+)$/,
    canParseFloatRe = /^[-+]?(\d[\d_]*)?\.[\d_]*(e[-+]\d+)?$/i,
    nullRe   = /^(~|null)$/i,
    trueRe   = /^(y|yes|true|on)$/i,
    falseRe  = /^(n|no|false|off)$/i,
    posInfRe = /^\+?\.inf$/i,
    negInfRe = /^-\.inf$/i,
    nanRe    = /^\.nan$/i,
    binRe    = /^([-+])?0b([01_]+)$/,
    timeRe   = /^([-+]?)([0-9][\d_]*(?::[0-5]?\d)+(?:\.[\d_]*)?)$/,
    dateRe   = /^\d{4}-\d\d?-\d\d?$/,
    timestampRe = /^(\d{4}-\d\d?-\d\d?(?:[Tt]|\s+)\d\d?:\d\d:\d\d(?:\.\d*)?)(?:\s*(Z|[-+]\d\d?(?::\d\d)?))?$/,
    underscoresRe = /_/g;

// Helper function that converts a string scalar to its actual type.
var parseScalar = function(v) {
  if (!v) return null;

  // Simple keywords.
  if (definitelyNonNumericRe.test(v)) {
    if (v.length > 5 || definitelyNonBooleanRe.test(v)) return v;
    if (nullRe.test(v)) return null;
    if (trueRe.test(v)) return true;
    if (falseRe.test(v)) return false;
    return v;
  }
  if (posInfRe.test(v)) return  1/0;
  if (negInfRe.test(v)) return -1/0;
  if (nanRe.test(v))    return  0/0;

  // JavaScript's `parseInt` does not support binary numbers.
  var m = binRe.exec(v);
  if (m) {
    var s = m[2],
        length = s.length,
        result = 0;
    for (var i = 0; i < length; i++) {
      if (s[i] == '_')
        continue;
      result *= 2;
      if (s[i] == '1')
        result++;
    }
    if (m[1] == '-')
      result *= -1;
    return result;
  }

  // JavaScript's datetime parsing is subtly different from YAML's.
  var m = timestampRe.exec(v);
  if (m) {
    var dateTimePart = m[1].replace('t', 'T'),
        offset = 0;
    if (m[2] && m[2] !== 'Z') {
      var parts = m[2].split(':');
      offset = parseInt(parts[0], 10) * 100;
      if (parts.length == 2)
        offset += parseInt(parts[1], 10);
    }
    if (offset >= 0)
      offset = "+" + offset;
    return new Date(dateTimePart + offset);
  }
  if (dateRe.test(v))
    return new Date(v + "T00:00:00Z");

  // Regular numbers.
  if (canParseIntRe.test(v))
    return parseInt(v.replace(underscoresRe, ''), 0);
  if (canParseFloatRe.test(v))
    return parseFloat(v.replace(underscoresRe, ''));

  // Times.
  var m = timeRe.exec(v);
  if (m) {
    var parts = m[2].split(':'),
        length = parts.length,
        result = 0;
    for (var i = 0; i < length; i++) {
      result *= 60;
      if (length == i + 1)
        result += parseFloat(parts[i]);
      else
        result += parseInt(parts[i], 10);
    }
    if (m[1] == '-')
      result *= -1;
    return result;
  }<commit_msg>Fix date parsing in new V8.<commit_after>// YAML.node, © 2011 <NAME>
// MIT-licensed. (See the included LICENSE file.)

var fs = require('fs'),
    binding = exports.capi = require('./build/Release/binding');


// Most of these were derived from: http://yaml.org/type/
// Also borrows from tenderlove's `Psych::ScalarScanner`. (MIT-licensed)
var definitelyNonNumericRe = /^[a-z~]/i,
    definitelyNonBooleanRe = /^[^ytonf~]/i,
    canParseIntRe   = /^[-+]?(0x[0-9a-fA-F_]+|[\d_]+)$/,
    canParseFloatRe = /^[-+]?(\d[\d_]*)?\.[\d_]*(e[-+]\d+)?$/i,
    nullRe   = /^(~|null)$/i,
    trueRe   = /^(y|yes|true|on)$/i,
    falseRe  = /^(n|no|false|off)$/i,
    posInfRe = /^\+?\.inf$/i,
    negInfRe = /^-\.inf$/i,
    nanRe    = /^\.nan$/i,
    binRe    = /^([-+])?0b([01_]+)$/,
    timeRe   = /^([-+]?)([0-9][\d_]*(?::[0-5]?\d)+(?:\.[\d_]*)?)$/,
    dateRe   = /^\d{4}-\d\d?-\d\d?$/,
    timestampRe = /^(\d{4}-\d\d?-\d\d?(?:[Tt]|\s+)\d\d?:\d\d:\d\d(?:\.\d*)?)(?:\s*(Z|[-+]\d\d?(?::\d\d)?))?$/,
    underscoresRe = /_/g;

// Tiny helper to pad a string.
var pad = function(s, num, ch) {
  while (s.length < num) s = ch + s;
  return s;
};

// Helper function that converts a string scalar to its actual type.
var parseScalar = function(v) {
  if (!v) return null;

  // Simple keywords.
  if (definitelyNonNumericRe.test(v)) {
    if (v.length > 5 || definitelyNonBooleanRe.test(v)) return v;
    if (nullRe.test(v)) return null;
    if (trueRe.test(v)) return true;
    if (falseRe.test(v)) return false;
    return v;
  }
  if (posInfRe.test(v)) return  1/0;
  if (negInfRe.test(v)) return -1/0;
  if (nanRe.test(v))    return  0/0;

  // JavaScript's `parseInt` does not support binary numbers.
  var m = binRe.exec(v);
  if (m) {
    var s = m[2],
        length = s.length,
        result = 0;
    for (var i = 0; i < length; i++) {
      if (s[i] == '_')
        continue;
      result *= 2;
      if (s[i] == '1')
        result++;
    }
    if (m[1] == '-')
      result *= -1;
    return result;
  }

  // JavaScript's datetime parsing is subtly different from YAML's.
  var m = timestampRe.exec(v);
  if (m) {
    var dateTimePart = m[1].replace('t', 'T'),
        offset = 0;
    if (m[2] && m[2] !== 'Z') {
      var parts = m[2].split(':');
      offset = parseInt(parts[0], 10) * 100;
      if (parts.length == 2)
        offset += parseInt(parts[1], 10);
    }
    if (offset >= 0)
      offset = "+" + pad(String(offset), 4, '0');
    else
      offset = "-" + pad(String(Math.abs(offset)), 4, '0');
    return new Date(dateTimePart + offset);
  }
  if (dateRe.test(v))
    return new Date(v + "T00:00:00Z");

  // Regular numbers.
  if (canParseIntRe.test(v))
    return parseInt(v.replace(underscoresRe, ''), 0);
  if (canParseFloatRe.test(v))
    return parseFloat(v.replace(underscoresRe, ''));

  // Times.
  var m = timeRe.exec(v);
  if (m) {
    var parts = m[2].split(':'),
        length = parts.length,
        result = 0;
    for (var i = 0; i < length; i++) {
      result *= 60;
      if (length == i + 1)
        result += parseFloat(parts[i]);
      else
        result += parseInt(parts[i], 10);
    }
    if (m[1] == '-')
      result *= -1;
    return result;
  }
{'fraction_non_alphanumeric': 0.23265239948119326, 'fraction_numerical': 0.021238651102464333, 'mean_word_length': 2.3454446854663775, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '27002', 'n_tokens_mistral': 2593, 'n_tokens_neox': 2475, 'n_words': 543}