<commit_before>/*!
 * Dox
 * Copyright (c) 2011 <NAME> <<EMAIL>>
 * MIT Licensed
 */

/*!
 * Module dependencies.
 */

var markdown = require('github-flavored-markdown').parse
  , escape = require('./utils').escape;

/**
 * Library version.
 */

exports.version = '0.1.2';

/**
 * Parse comments in the given string of `js`.
 *
 * @param {String} js
 * @return {Array}
 * @see exports.parseComment
 * @api public
 */

exports.parseComments = function(js){
  var comments = []
    , comment
    , buf = ''
    , ignore
    , within
    , code;

  for (var i = 0, len = js.length; i < len; ++i) {
    // start comment
    if (!within && '/' == js[i] && '*' == js[i+1]) {
      // code following previous comment
      if (buf.trim().length) {
        comment = comments[comments.length - 1];
        if(comment) {
	  comment.code = code = buf.trim();
	  comment.ctx = exports.parseCodeContext(code);
        }
        buf = '';
      }
      i += 2;
      within = true;
      ignore = '!' == js[i];
    // end comment
    } else if (within && '*' == js[i] && '/' == js[i+1]) {
      i += 2;
      buf = buf.replace(/^ *\* ?/gm, '');
      var comment = exports.parseComment(buf);
      comment.ignore = ignore;
      comments.push(comment);
      within = ignore = false;
      buf = '';
    // buffer comment or code
    } else {
      buf += js[i];
    }<commit_msg>tabs replaced to spaces<commit_after>/*!
 * Dox
 * Copyright (c) 2011 <NAME> <<EMAIL>>
 * MIT Licensed
 */

/*!
 * Module dependencies.
 */

var markdown = require('github-flavored-markdown').parse
  , escape = require('./utils').escape;

/**
 * Library version.
 */

exports.version = '0.1.2';

/**
 * Parse comments in the given string of `js`.
 *
 * @param {String} js
 * @return {Array}
 * @see exports.parseComment
 * @api public
 */

exports.parseComments = function(js){
  var comments = []
    , comment
    , buf = ''
    , ignore
    , within
    , code;

  for (var i = 0, len = js.length; i < len; ++i) {
    // start comment
    if (!within && '/' == js[i] && '*' == js[i+1]) {
      // code following previous comment
      if (buf.trim().length) {
        comment = comments[comments.length - 1];
        if(comment) {
          comment.code = code = buf.trim();
          comment.ctx = exports.parseCodeContext(code);
        }
        buf = '';
      }
      i += 2;
      within = true;
      ignore = '!' == js[i];
    // end comment
    } else if (within && '*' == js[i] && '/' == js[i+1]) {
      i += 2;
      buf = buf.replace(/^ *\* ?/gm, '');
      var comment = exports.parseComment(buf);
      comment.ignore = ignore;
      comments.push(comment);
      within = ignore = false;
      buf = '';
    // buffer comment or code
    } else {
      buf += js[i];
    }
{'fraction_non_alphanumeric': 0.18709209572153734, 'fraction_numerical': 0.009427121102248005, 'mean_word_length': 2.0587583148558757, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3888566', 'n_tokens_mistral': 936, 'n_tokens_neox': 898, 'n_words': 227}