<commit_before>'use strict';
var _ = require('lodash');
var gutil = require('gulp-util');
var path = require('path');
var docco = require('docco');
var HtmlParser = require('htmlparser2').Parser;

var defaultOptions = {
  htmlParserExt: ['.html', '.xml']
};

function countLines(str) {
  return str.split("\n").length - 1;
}

function tokenizeHtml(code) {
  var parser, lastComment = null, tokens = [];

  function pushComment(text) {
    tokens.push({ type: 'comment', value: text.replace(/^[\t ]|[\t ]$/g, '') });
  }
  function pushCode(start, end) {
    tokens.push({ type: 'code', value: code.substr(start, end - start) });
  }
  function flushCode(comment) {
    var lastEnd = lastComment ? lastComment.end + 1 : 0;
    if (comment.start !== lastEnd || !code.length)
      pushCode(lastEnd, comment.start);
  }
  function onComment(text) {
    var comment = { start: parser.startIndex, end: parser.endIndex };
    flushCode(comment);
    pushComment(text);
    lastComment = comment;
  }

  parser = new HtmlParser({ oncomment: onComment });

  parser.write(code);
  parser.end();

  flushCode({start: code.length});

  return tokens;
}

function Formatter() {
  this.list = [];
  this.currentLine = 1;
  this.currentSection = null;
}

Formatter.prototype.addToLines = function (count) {
  this.currentLine += count;
};

Formatter.prototype.addToken = function (token) {
  var needsSection =
      !this.currentSection            // First time here
    || this.currentSection.codeSet    // Code set means no more tokens,
                                      //     comments come before Code.
    || token.type === 'comment'       // Comments are set once.
      && this.currentSection.docsSet;

  if (needsSection)
    this.currentSection = new Section(this);

  this.currentSection.add(token);
  return this;
};

function Section(parent) {
  this.parent = parent;
  this.attrs = {
    docsText: '',
    docsLine: 1,
    codeText: '',
    codeLine: 1
  };
  this.docsSet = false;
  this.codeSet = false;
  this.parent.list.push(this.attrs);
}

Section.prototype.add = function (token) {
  if (token.type === 'code') {
    this.attrs.codeText = token.value;
    this.attrs.codeLine = this.parent.currentLine;
    this.codeSet = true;
    this.parent.addToLines(countLines(token.value));
  } else {
    this.attrs.docsText = token.value;
    this.attrs.docsLine = this.parent.currentLine;
    this.docsSet = true;
    this.parent.addToLines(countLines(token.value));
    this.attrs.codeLine = this.parent.currentLine;
  }
  return this;
};

function partWithDocco(pathName, code, options) {
  if (!options.languages) options.languages = {}
  var sections = docco.parse(pathName, code, options);
  var startLine = 1;
  _.forEach(sections, function (section) {
    section.docsLine = startLine;
    startLine += countLines(section.docsText);
    section.codeLine = startLine;
    startLine += countLines(section.codeText);
  });
  return sections;
}

function partWithHtmlParser(pathName, code, options) {
  var tokens = tokenizeHtml(code, options);
  var formatter = new Formatter();
  _.forEach(tokens, function (token) {
    formatter.addToken(token);
  });
  return formatter.list;
}

function part(pathName, code, options) {
  var ext = options.extname || path.extname(pathName);
  if (options.htmlParserExt.indexOf(ext.toLowerCase()) != -1)
    return partWithHtmlParser(pathName, code, options);
  else
    return partWithDocco(pathName, code, options);
}

module.exports = function (pathName, source, options) {
  if (!pathName) throw new Error("path not specified to code-comb");
  if (!options) options = {};<commit_msg>removed debugging require and cleaned up for jshint<commit_after>'use strict';
var _ = require('lodash');
var path = require('path');
var docco = require('docco');
var HtmlParser = require('htmlparser2').Parser;

var defaultOptions = {
  htmlParserExt: ['.html', '.xml']
};

function countLines(str) {
  return str.split("\n").length - 1;
}

function tokenizeHtml(code) {
  var parser, lastComment = null, tokens = [];

  function pushComment(text) {
    tokens.push({ type: 'comment', value: text.replace(/^[\t ]|[\t ]$/g, '') });
  }
  function pushCode(start, end) {
    tokens.push({ type: 'code', value: code.substr(start, end - start) });
  }
  function flushCode(comment) {
    var lastEnd = lastComment ? lastComment.end + 1 : 0;
    if (comment.start !== lastEnd || !code.length)
      pushCode(lastEnd, comment.start);
  }
  function onComment(text) {
    var comment = { start: parser.startIndex, end: parser.endIndex };
    flushCode(comment);
    pushComment(text);
    lastComment = comment;
  }

  parser = new HtmlParser({ oncomment: onComment });

  parser.write(code);
  parser.end();

  flushCode({start: code.length});

  return tokens;
}

function Formatter() {
  this.list = [];
  this.currentLine = 1;
  this.currentSection = null;
}

Formatter.prototype.addToLines = function (count) {
  this.currentLine += count;
};

Formatter.prototype.addToken = function (token) {
  var needsSection =
    // First time here
    !this.currentSection ||
    // Code set means no more tokens,
    // comments come before Code.    
    this.currentSection.codeSet ||
    // Comments are set once.
    token.type === 'comment' &&
    this.currentSection.docsSet;

  if (needsSection)
    this.currentSection = new Section(this);

  this.currentSection.add(token);
  return this;
};

function Section(parent) {
  this.parent = parent;
  this.attrs = {
    docsText: '',
    docsLine: 1,
    codeText: '',
    codeLine: 1
  };
  this.docsSet = false;
  this.codeSet = false;
  this.parent.list.push(this.attrs);
}

Section.prototype.add = function (token) {
  if (token.type === 'code') {
    this.attrs.codeText = token.value;
    this.attrs.codeLine = this.parent.currentLine;
    this.codeSet = true;
    this.parent.addToLines(countLines(token.value));
  } else {
    this.attrs.docsText = token.value;
    this.attrs.docsLine = this.parent.currentLine;
    this.docsSet = true;
    this.parent.addToLines(countLines(token.value));
    this.attrs.codeLine = this.parent.currentLine;
  }
  return this;
};

function partWithDocco(pathName, code, options) {
  if (!options.languages) options.languages = {};
  var sections = docco.parse(pathName, code, options);
  var startLine = 1;
  _.forEach(sections, function (section) {
    section.docsLine = startLine;
    startLine += countLines(section.docsText);
    section.codeLine = startLine;
    startLine += countLines(section.codeText);
  });
  return sections;
}

function partWithHtmlParser(pathName, code, options) {
  var tokens = tokenizeHtml(code, options);
  var formatter = new Formatter();
  _.forEach(tokens, function (token) {
    formatter.addToken(token);
  });
  return formatter.list;
}

function part(pathName, code, options) {
  var ext = options.extname || path.extname(pathName);
  if (options.htmlParserExt.indexOf(ext.toLowerCase()) != -1)
    return partWithHtmlParser(pathName, code, options);
  else
    return partWithDocco(pathName, code, options);
}

module.exports = function (pathName, source, options) {
  if (!pathName) throw new Error("path not specified to code-comb");
  if (!options) options = {};
{'fraction_non_alphanumeric': 0.14422145328719724, 'fraction_numerical': 0.002491349480968858, 'mean_word_length': 3.599618077657543, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4371279', 'n_tokens_mistral': 2312, 'n_tokens_neox': 2274, 'n_words': 524}