<commit_before>'use strict';

var debug = require('debug')('licenses::content');

module.exports = require('./parser').extend({
  /**
   * Parse the markdown information from the package.
   *
   * @param {Object} data The package.json or npm package contents.
   * @param {Function} next Continuation.
   * @api public
   */
  parse: function parse(data, next) {
    data = this.get(data);

    //
    // We cannot detect a license so we call the callback without any arguments
    // which symbolises a failed attempt.
    //
    if (!data) return next();

    //
    // Optimize the matches by trying to locate where the licensing information
    // starts in the given content. Usually, we, as developers add it at the
    // bottom of our README.md files and prefix it with "LICENSE" as header.
    //
    data.split('\n').forEach(function some(line, index, lines) {
      if (/^.{0,7}\s{0,}(?:licen[cs]e[s]?|copyright).{0,2}\s{0,}$/gim.test(line.trim())) {
        data = lines.slice(index).join('\n');
        debug('matched %s as license header, slicing data', JSON.stringify(line));
      }
    });

    var license = this.scan(data);
    if (!license) {
      license = this.test(data);

      if (license) debug('used regexp to detect %s in content', license);<commit_msg>[minor] Just use first occurance<commit_after>'use strict';

var debug = require('debug')('licenses::content');

module.exports = require('./parser').extend({
  /**
   * Parse the markdown information from the package.
   *
   * @param {Object} data The package.json or npm package contents.
   * @param {Function} next Continuation.
   * @api public
   */
  parse: function parse(data, next) {
    data = this.get(data);

    //
    // We cannot detect a license so we call the callback without any arguments
    // which symbolises a failed attempt.
    //
    if (!data) return next();

    //
    // Optimize the matches by trying to locate where the licensing information
    // starts in the given content. Usually, we, as developers add it at the
    // bottom of our README.md files and prefix it with "LICENSE" as header.
    //
    data.split('\n').some(function some(line, index, lines) {
      if (/^.{0,7}\s{0,}(?:licen[cs]e[s]?|copyright).{0,2}\s{0,}$/gim.test(line.trim())) {
        data = lines.slice(index).join('\n');
        debug('matched %s as license header, slicing data', JSON.stringify(line));
        return true;
      }

      return false;
    });

    var license = this.scan(data);
    if (!license) {
      license = this.test(data);

      if (license) debug('used regexp to detect %s in content', license);
{'fraction_non_alphanumeric': 0.1486280487804878, 'fraction_numerical': 0.004573170731707317, 'mean_word_length': 2.826530612244898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1238340', 'n_tokens_mistral': 790, 'n_tokens_neox': 774, 'n_words': 261}