<commit_before>var utils = require('./imap.utilities');

var reCRLF = /\r\n/g,
    reHdr = /^([^:]+):\s(.+)?$/,
    reHdrFold = /^\s+(.+)$/;

exports.convStr = function(str, literals) {
  if (str[0] === '"')
    return str.substring(1, str.length-1);
  else if (str === 'NIL')
    return null;
  else if (/^\d+$/.test(str)) {
    // some IMAP extensions utilize large (64-bit) integers, which JavaScript
    // can't handle natively, so we'll just keep it as a string if it's too big
    var val = parseInt(str, 10);
    return (val.toString() === str ? val : str);
  } else if (literals && literals.lp < literals.length && /^\{\d+\}$/.test(str))
    return literals[literals.lp++];
  else
    return str;
};

exports.parseHeaders = function(str) {
  var lines = str.split(reCRLF),
      headers = {}, m;

  for (var i = 0, h, len = lines.length; i < len; ++i) {
    if (lines[i].length === 0)
      continue;
    if (lines[i][0] === '\t' || lines[i][0] === ' ') {
      // folded header content
      m = reHdrFold.exec(lines[i]);
      headers[h][headers[h].length - 1] += m[1];
    } else {
      m = reHdr.exec(lines[i]);
      h = m[1].toLowerCase();
      if (m[2]) {
        if (headers[h] === undefined)
          headers[h] = [m[2]];
        else
          headers[h].push(m[2]);
      } else
        headers[h] = [''];
    }
  }
  return headers;
};

exports.parseNamespaces = function(str, literals) {
  var result, vals;
  if (str.length === 3 && str.toUpperCase() === 'NIL')
    vals = null;
  else {
    result = exports.parseExpr(str, literals);
    vals = [];
    for (var i=0,len=result.length; i<len; ++i) {
      var val = {
        prefix: result[i][0],
        delimiter: result[i][1]
      };
      if (result[i].length > 2) {
        // extension data
        val.extensions = [];
        for (var j=2,len2=result[i].length; j<len2; j+=2) {
          val.extensions.push({
            name: result[i][j],
            flags: result[i][j+1]
          });
        }
      }
      vals.push(val);
    }
  }
  return vals;
};

exports.parseFetchBodies = function(str, literals) {
  literals.lp = 0;
  var result = exports.parseExpr(str, literals),
      bodies;
  for (var i=0,len=result.length; i<len; i+=2) {
    if (Array.isArray(result[i])) {
      if (result[i].length > 1) {
        // HEADER.FIELDS (foo)
        result[i][0] += ' (';
        result[i][0] += result[i].slice(1).join(' ');
        result[i][0] += ')';
      }
      if (bodies === undefined)
        bodies = ['BODY[' + result[i][0] + ']', result[i+1]];
      else {
        bodies.push('BODY[' + result[i][0] + ']');
        bodies.push(result[i+1]);
      }<commit_msg>fix BODY[] parsing<commit_after>var utils = require('./imap.utilities');

var reCRLF = /\r\n/g,
    reHdr = /^([^:]+):\s(.+)?$/,
    reHdrFold = /^\s+(.+)$/;

exports.convStr = function(str, literals) {
  if (str[0] === '"')
    return str.substring(1, str.length-1);
  else if (str === 'NIL')
    return null;
  else if (/^\d+$/.test(str)) {
    // some IMAP extensions utilize large (64-bit) integers, which JavaScript
    // can't handle natively, so we'll just keep it as a string if it's too big
    var val = parseInt(str, 10);
    return (val.toString() === str ? val : str);
  } else if (literals && literals.lp < literals.length && /^\{\d+\}$/.test(str))
    return literals[literals.lp++];
  else
    return str;
};

exports.parseHeaders = function(str) {
  var lines = str.split(reCRLF),
      headers = {}, m;

  for (var i = 0, h, len = lines.length; i < len; ++i) {
    if (lines[i].length === 0)
      continue;
    if (lines[i][0] === '\t' || lines[i][0] === ' ') {
      // folded header content
      m = reHdrFold.exec(lines[i]);
      headers[h][headers[h].length - 1] += m[1];
    } else {
      m = reHdr.exec(lines[i]);
      h = m[1].toLowerCase();
      if (m[2]) {
        if (headers[h] === undefined)
          headers[h] = [m[2]];
        else
          headers[h].push(m[2]);
      } else
        headers[h] = [''];
    }
  }
  return headers;
};

exports.parseNamespaces = function(str, literals) {
  var result, vals;
  if (str.length === 3 && str.toUpperCase() === 'NIL')
    vals = null;
  else {
    result = exports.parseExpr(str, literals);
    vals = [];
    for (var i=0,len=result.length; i<len; ++i) {
      var val = {
        prefix: result[i][0],
        delimiter: result[i][1]
      };
      if (result[i].length > 2) {
        // extension data
        val.extensions = [];
        for (var j=2,len2=result[i].length; j<len2; j+=2) {
          val.extensions.push({
            name: result[i][j],
            flags: result[i][j+1]
          });
        }
      }
      vals.push(val);
    }
  }
  return vals;
};

exports.parseFetchBodies = function(str, literals) {
  literals.lp = 0;
  var result = exports.parseExpr(str, literals),
      bodies;
  for (var i=0,len=result.length; i<len; i+=2) {
    if (Array.isArray(result[i])) {
      if (result[i].length === 0)
        result[i].push('');
      else if (result[i].length > 1) {
        // HEADER.FIELDS (foo)
        result[i][0] += ' (';
        result[i][0] += result[i].slice(1).join(' ');
        result[i][0] += ')';
      }
      if (bodies === undefined)
        bodies = ['BODY[' + result[i][0] + ']', result[i+1]];
      else {
        bodies.push('BODY[' + result[i][0] + ']');
        bodies.push(result[i+1]);
      }
{'fraction_non_alphanumeric': 0.2079318013343217, 'fraction_numerical': 0.01464047442550037, 'mean_word_length': 2.2336728579988017, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1505345', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1979, 'n_words': 421}