<commit_before>'use strict'

var kssBlocksParser = require('./kss-blocks-parser').kssBlocksParser;

module.exports = {

  /* Split string source into array of code and comment blocks */
  pureSplitter: function(source) {
    var lines = source.split(/\n|$/g),
      isAnInlineComment = / *\/\//,
      blocks = [],
      block,
      prevType;

    lines.forEach(function(line){

      var type = isAnInlineComment.test(line)? 'comment' : 'code';

      if (prevType != type) {
        // Save old block if a new type appears
        if (block) {
          block.content = block.content.join('\n');
          blocks.push(block);
        }
        // Start a new block after changing the type
        block = {
          type: type,
          content: [line]
        }
      } else {
        block.content.push(line);
      }

      prevType = type;

    });

    // Push the last block
    block.content = block.content.join('\n');
    blocks.push(block);

    return blocks;
  },

  getBlocks: function(source) {

    var blocks = this.pureSplitter(source),
      pair = {
        kss: '',
        code: []
      },
      pairs = [],
      isKssMarkupBlock = /Styleguide [0-9]+/

    blocks.forEach(function(block){

      if (block.type == 'comment') {
        // Check if KSS
        if (isKssMarkupBlock.test(block.content)) {
          // Save old pair
          if (pair) {
            pair.code = pair.code.join('');
            pairs.push(pair)
          }
          // KSS starts a new block
          pair = {
            kss: block.content,
            code: []
          }
        } else {
          // Not KSS comments are considered to be parts of code
          pair.code.push('\n')
          pair.code.push(block.content)
        }
      } else {
        pair.code.push('\n')
        pair.code.push(block.content)
      }

    });

    // Push last pair
    pair.code = pair.code.join('');
    pairs.push(pair);

    return pairs;
  }

}
<commit_msg>Do not save first empty block into pair<commit_after>'use strict'

var kssBlocksParser = require('./kss-blocks-parser').kssBlocksParser;

module.exports = {

  /* Split string source into array of code and comment blocks */
  pureSplitter: function(source) {
    var lines = source.split(/\n|$/g),
      isAnInlineComment = / *\/\//,
      blocks = [],
      block,
      prevType;

    lines.forEach(function(line){

      var type = isAnInlineComment.test(line)? 'comment' : 'code';

      if (prevType != type) {
        // Save old block if a new type appears
        if (block) {
          block.content = block.content.join('\n');
          blocks.push(block);
        }
        // Start a new block after changing the type
        block = {
          type: type,
          content: [line]
        }
      } else {
        block.content.push(line);
      }

      prevType = type;

    });

    // Push the last block
    block.content = block.content.join('\n');
    blocks.push(block);

    return blocks;
  },

  getBlocks: function(source) {

    var blocks = this.pureSplitter(source),
      pair = {
        kss: '',
        code: []
      },
      firstBlock = true,
      pairs = [],
      isKssMarkupBlock = /Styleguide [0-9]+/

    blocks.forEach(function(block){

      if (block.type == 'comment') {
        // Check if KSS
        if (isKssMarkupBlock.test(block.content)) {
          // Save old pair
          if (pair && !firstBlock) {
            pair.code = pair.code.join('');
            pairs.push(pair)
          }
          // KSS starts a new block
          pair = {
            kss: block.content,
            code: []
          }
        } else {
          // Not KSS comments are considered to be parts of code
          pair.code.push('\n')
          pair.code.push(block.content)
        }
      } else {
        pair.code.push('\n')
        pair.code.push(block.content)
      }

      firstBlock = false;

    });

    // Push last pair
    pair.code = pair.code.join('');
    pairs.push(pair);

    return pairs;
  }

}

{'fraction_non_alphanumeric': 0.13590062111801243, 'fraction_numerical': 0.0009937888198757764, 'mean_word_length': 1.6092028515878158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4303550', 'n_tokens_mistral': 1237, 'n_tokens_neox': 1215, 'n_words': 282}