<commit_before>'use strict';

var through2 = require('through2');
var fs = require('fs');
var path = require('path');
var _ = require('lodash');
var beautifyHtml = require('js-beautify').html;
var gutil = require('gulp-util');
var cheerio = require('cheerio');

function combo (opts) {
  var config = _.assign({
    app: null,
    module: null,
    cwd: null,
    fdPath: null,
    domain: null,
    needCombo: false
  }, opts);
  config.comboPrefix = opts.comboPrefix ? opts.comboPrefix : '/c/=';
  var modulePath = path.join(config.cwd, config.module);

  var stream = through2.obj(function (file, encoding, callback) {
    if (file.isNull()) {
      return callback(null, file);
    }
    if (file.isBuffer()) {
      if (config.fdPath.lastIndexOf('/') === 0) {
        config.fdPath = config.fdPath + '/';
      }
      var filePathObj = path.parse(file.path);
      var fileName = filePathObj.name;
      var fileContent = file.contents.toString();
      var newFileContent = '';
      var $ = cheerio.load(fileContent, { decodeEntities: false });
      var headChildren = $('head').contents();

      headChildren.each(function (i, item) {
        var nodeType = item.nodeType;
        if (nodeType === 8 && /global|combo/.test(item.data)) {
          $(item).remove();
        }

      });
      var links = $('link[rel=stylesheet]');
      if (config.needCombo) {
        var combofile = config.fdPath + config.app + '/' + config.module + '/' + fileName + '.shtml';
        var href = '//' + config.domain + config.comboPrefix;
        links.each(function (i, item) {<commit_msg>fix:修复页面片占位被干掉的问题<commit_after>'use strict';

var through2 = require('through2');
var fs = require('fs');
var path = require('path');
var _ = require('lodash');
var beautifyHtml = require('js-beautify').html;
var gutil = require('gulp-util');
var cheerio = require('cheerio');

function combo (opts) {
  var config = _.assign({
    app: null,
    module: null,
    cwd: null,
    fdPath: null,
    domain: null,
    needCombo: false
  }, opts);
  config.comboPrefix = opts.comboPrefix ? opts.comboPrefix : '/c/=';
  var modulePath = path.join(config.cwd, config.module);

  var stream = through2.obj(function (file, encoding, callback) {
    if (file.isNull()) {
      return callback(null, file);
    }
    if (file.isBuffer()) {
      if (config.fdPath.lastIndexOf('/') === 0) {
        config.fdPath = config.fdPath + '/';
      }
      var filePathObj = path.parse(file.path);
      var fileName = filePathObj.name;
      var fileContent = file.contents.toString();
      var newFileContent = '';
      var $ = cheerio.load(fileContent, { decodeEntities: false });
      var headChildren = $('head').contents();

      headChildren.each(function (i, item) {
        var nodeType = item.nodeType;
        if (nodeType === 8 && /global:|endglobal/.test(item.data)) {
          $(item).remove();
        }
      });
      var links = $('link[rel=stylesheet]');
      if (config.needCombo) {
        var combofile = config.fdPath + config.app + '/' + config.module + '/' + fileName + '.shtml';
        var href = '//' + config.domain + config.comboPrefix;
        links.each(function (i, item) {
{'fraction_non_alphanumeric': 0.16656191074795726, 'fraction_numerical': 0.0031426775612822125, 'mean_word_length': 2.8349397590361445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2216995', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1029, 'n_words': 230}