<commit_before>
var esprima = require('esprima');


/**
 * Position in the source file.
 *
 * @see [SpiderMonkey Parser API]
 *   {@link https://developer.mozilla.org/en-US/docs/Mozilla/Projects/SpiderMonkey/Parser_API}
 *
 * @typedef {Object} Position
 * @property {number} line - Line number, 1-indexed.
 * @property {?number} column - Column number, 0-indexed.
 */


/**
 * Location - position range.
 *
 * @typedef {Object} Location
 * @property {Position} start
 * @property {Position} end
 */


/**
 * Extract comments from code.
 * Each comment is represented by a string within its usual comment delimiters.
 * Returned location always contains both line and column info.
 *
 * @arg {string} code - JavaScript code.
 * @return {{text: string, loc: Location}[]}
 */
var extract = function (code) {
  return esprima.parse(code, {
    comment: true,
    loc: true
  }).comments.map(function (comment) {
    return {
      text: (comment.type == 'Line')
        ? '//' + comment.value
        : '/*' + comment.value + '*/',
      loc: comment.loc
    };
  });
};
<commit_msg>Make column numbers in Location mandatory<commit_after>var esprima = require('esprima');


/**
 * Position in the source file.
 *
 * @see [SpiderMonkey Parser API]
 *   {@link https://developer.mozilla.org/en-US/docs/Mozilla/Projects/SpiderMonkey/Parser_API}
 *
 * @typedef {Object} Position
 * @property {number} line - Line number, 1-indexed.
 * @property {?number} column - Column number, 0-indexed.
 */


/**
 * Location - position range.
 *
 * @typedef {Object} Location
 * @property {Position} start - Column number must be set.
 * @property {Position} end - Column number must be set.
 */


/**
 * Extract comments from code.
 * Each comment is represented by a string within its usual comment delimiters.
 *
 * @arg {string} code - JavaScript code.
 * @return {{text: string, loc: Location}[]}
 */
var extract = function (code) {
  return esprima.parse(code, {
    comment: true,
    loc: true
  }).comments.map(function (comment) {
    return {
      text: (comment.type == 'Line')
        ? '//' + comment.value
        : '/*' + comment.value + '*/',
      loc: comment.loc
    };
  });
};

{'fraction_non_alphanumeric': 0.15328132170720515, 'fraction_numerical': 0.0018357044515832951, 'mean_word_length': 3.0295748613678373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5061786', 'n_tokens_mistral': 704, 'n_tokens_neox': 681, 'n_words': 199}