<commit_before>var rLinkScript = /(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(?:(\s*<script([^>]*)>([\s\S]*?)<\/script>)|(?:\s*(<link([^>]*?)(?:\/)?>)|(<style([^>]*)>([\s\S]*?)<\/style>)))(<!--ignore-->)?\n?/ig;
// var rScript = /(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(\s*<script([^>]*)>([\s\S]*?)<\/script>)(<!--ignore-->)?\n?/ig;
var rScriptType = /type=('|")(.*?)\1/i;
var rSrcHref = /\s*(?:src|href)=('|")(.+?)\1/i;
// var rStyle = /(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(?:\s*(<link([^>]*?)(?:\/)?>)|(<style([^>]*)>([\s\S]*?)<\/style>))(<!--ignore-->)?\n?/ig;
var rRefStyle = /rel=('|")stylesheet\1/i;
var rLoader = /data\-loader(?:=('|").*?\1)?/i;
var rFramework = /data\-framework(?:=('|").*?\1)?/i;
var common = require('../common.js');
var rHead = /<!--([\s\S]+?)(?:-->|$)|<\/head>/ig;
var rBody = /<!--([\s\S]+?)(?:-->|$)|<\/body>/ig;
var path = require('path');

/**
 * 查找 script，找到前端 loader 特殊处理。
 *
 * 以下两种 case 都被认为是前端 loader。
 * 1. mod.js、esl.js或者 require.js
 * 2. <script> 标签带有 data-loader 属性。
 */
function obtainFramework(content, resource, opts, host) {
  rLinkScript.lastIndex = rLoader.lastIndex = 0;

  return content.replace(rLinkScript, function(all, comment, script, attrs, body, link, lattrs, style, sattrs, sbody, ignored) {

    // 忽略注释。
    if (comment || ignored) {
      return all;
    }

    if (script && !body.trim() && rSrcHref.test(attrs)) {
      var src = RegExp.$2;
      var file = resource.getFileByUrl(src);

      attrs = attrs.replace(rSrcHref, '').replace(/\s+$/, '');

      if (!file) {
        file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), src)));
      }

      // 如果 require.js 或者 esl.js 或者是 mod.js
      // 或者 <script 设置了 data-loader 属性
      // 则认为此资源为前端loader
      if (rLoader.test(attrs) || file && ~opts.loaderScripts.indexOf(file.basename)) {

        // 根据 js 自动设置 resourceType。
        if (opts.resourceType === 'auto' && file) {
          opts.resourceType = file.basename === 'mod.js' ? 'mod' :
            (file.basename === 'sea.js' ? 'cmd' :
              (file.basename === 'system.js' ? 'system' : 'amd'));
        }

        if (opts.obtainScript) {
          file ? resource.add(file.id, false, false, attrs) : resource.addJs(src, attrs);

          if (opts.useInlineMap) {
            if (!~content.indexOf(opts.resourcePlaceHolder)) {
              var idx = common.search(resource.res, function(item) {
                return item.content === ('/*resourcemap*/\n'+opts.resourcePlaceHolder);
              });
              ~idx && resource.res.splice(idx, 1);

              resource.addJsEmbed('/*resourcemap*/\n'+opts.resourcePlaceHolder);
            }
          } else if (!~content.indexOf(opts.resourcePlaceHolder)) {
            var idx = common.search(resource.res, function(item) {
              return item.uri === 'resourcePlaceHolder';
            });
            ~idx && resource.res.splice(idx, 1);

            resource.addJs('resourcePlaceHolder');
          }

          all = '';
        } else if (!~content.indexOf(opts.resourcePlaceHolder)) {
          all += '\n' + opts.resourcePlaceHolder;
        }

      } else if (rFramework.test(script) && opts.obtainScript) {
        file ? resource.add(file.id, false, false, attrs) : resource.addJs(src, attrs);
      }
    }

    return all;
  });
}

function obtainScriptAndStyle(content, resource, opts, host, includeList) {
  rLinkScript.lastIndex = 0;

  var obtainStyle = ~content.indexOf(opts.stylePlaceHolder) && opts.obtainStyle;
  var obtainScript = ~content.indexOf(opts.scriptPlaceHolder) && opts.obtainScript;

  return content.replace(rLinkScript, function(all, comment, script, attrs, body, link, lattrs, style, sattrs, sbody, ignored) {

    if (comment && ~comment.indexOf(opts.dependenciesInjectPlaceHolder)) {
      host.requires.forEach(function(id) {
        resource.add(id);
      });

      host.asyncs.forEach(function(id) {
        resource.add(id, true);
      });

      includeList.forEach(function(file) {
        resource.add(file.id, true);
      });

      return '';
    } else if (comment) {
      return all;
    } else if (script && !obtainScript) {
      return all;
    } else if ((link || style) && !obtainStyle) {
      return all;
    } else if (ignored) {
      // 如果是 ignored，至少应该把它原来的打包的路径替换掉。
      if (script && !body.trim() && rSrcHref.test(attrs)) {
        var src = RegExp.$2;
        var file = resource.getFileByUrl(src);
        attrs = attrs.replace(rSrcHref, '').replace(/\s+$/, '');

        if (!file) {
          file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), src)));
        }

        if (file) {
          const url = resource.getUri(file.id, true);
          all = script.replace(rSrcHref, ' src="' + url + '"');
        }
      } else if (link && rRefStyle.test(lattrs) && rSrcHref.test(lattrs)) {
        var href = RegExp.$2;
        var file = resource.getFileByUrl(href);
        lattrs = lattrs.replace(rSrcHref, '').replace(/\s+$/, '');

        if (!file) {
          file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), href)));
        }

        if (file) {
          const url = resource.getUri(file.id, true);
          all = link.replace(rSrcHref, ' href="' + url + '"');
        }
      }

      return all;
    }

    if (script && !body.trim() && rSrcHref.test(attrs)) {
      var src = RegExp.$2;
      var file = resource.getFileByUrl(src);
      attrs = attrs.replace(rSrcHref, '').replace(/\s+$/, '');

      if (!file) {
        file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), src)));
      }

      file ? resource.add(file.id, false, false, attrs) : resource.addJs(src, attrs);
      all = '';
    } else if (script && !rScriptType.test(attrs) || rScriptType.test(attrs) && ~['text/javascript', 'application/javascript'].indexOf(RegExp.$2.toLowerCase())) {
      resource.addJsEmbed(body, attrs);
      all = '';
    } else if (link && rRefStyle.test(lattrs) && rSrcHref.test(lattrs)) {
      var href = RegExp.$2;
      var file = resource.getFileByUrl(href);
      lattrs = lattrs.replace(rSrcHref, '').replace(/\s+$/, '');

      if (!file) {<commit_msg>fix const relative issue on some node's version<commit_after>var rLinkScript = /(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(?:(\s*<script([^>]*)>([\s\S]*?)<\/script>)|(?:\s*(<link([^>]*?)(?:\/)?>)|(<style([^>]*)>([\s\S]*?)<\/style>)))(<!--ignore-->)?\n?/ig;
// var rScript = /(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(\s*<script([^>]*)>([\s\S]*?)<\/script>)(<!--ignore-->)?\n?/ig;
var rScriptType = /type=('|")(.*?)\1/i;
var rSrcHref = /\s*(?:src|href)=('|")(.+?)\1/i;
// var rStyle = /(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(?:\s*(<link([^>]*?)(?:\/)?>)|(<style([^>]*)>([\s\S]*?)<\/style>))(<!--ignore-->)?\n?/ig;
var rRefStyle = /rel=('|")stylesheet\1/i;
var rLoader = /data\-loader(?:=('|").*?\1)?/i;
var rFramework = /data\-framework(?:=('|").*?\1)?/i;
var common = require('../common.js');
var rHead = /<!--([\s\S]+?)(?:-->|$)|<\/head>/ig;
var rBody = /<!--([\s\S]+?)(?:-->|$)|<\/body>/ig;
var path = require('path');

/**
 * 查找 script，找到前端 loader 特殊处理。
 *
 * 以下两种 case 都被认为是前端 loader。
 * 1. mod.js、esl.js或者 require.js
 * 2. <script> 标签带有 data-loader 属性。
 */
function obtainFramework(content, resource, opts, host) {
  rLinkScript.lastIndex = rLoader.lastIndex = 0;

  return content.replace(rLinkScript, function(all, comment, script, attrs, body, link, lattrs, style, sattrs, sbody, ignored) {

    // 忽略注释。
    if (comment || ignored) {
      return all;
    }

    if (script && !body.trim() && rSrcHref.test(attrs)) {
      var src = RegExp.$2;
      var file = resource.getFileByUrl(src);

      attrs = attrs.replace(rSrcHref, '').replace(/\s+$/, '');

      if (!file) {
        file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), src)));
      }

      // 如果 require.js 或者 esl.js 或者是 mod.js
      // 或者 <script 设置了 data-loader 属性
      // 则认为此资源为前端loader
      if (rLoader.test(attrs) || file && ~opts.loaderScripts.indexOf(file.basename)) {

        // 根据 js 自动设置 resourceType。
        if (opts.resourceType === 'auto' && file) {
          opts.resourceType = file.basename === 'mod.js' ? 'mod' :
            (file.basename === 'sea.js' ? 'cmd' :
              (file.basename === 'system.js' ? 'system' : 'amd'));
        }

        if (opts.obtainScript) {
          file ? resource.add(file.id, false, false, attrs) : resource.addJs(src, attrs);

          if (opts.useInlineMap) {
            if (!~content.indexOf(opts.resourcePlaceHolder)) {
              var idx = common.search(resource.res, function(item) {
                return item.content === ('/*resourcemap*/\n'+opts.resourcePlaceHolder);
              });
              ~idx && resource.res.splice(idx, 1);

              resource.addJsEmbed('/*resourcemap*/\n'+opts.resourcePlaceHolder);
            }
          } else if (!~content.indexOf(opts.resourcePlaceHolder)) {
            var idx = common.search(resource.res, function(item) {
              return item.uri === 'resourcePlaceHolder';
            });
            ~idx && resource.res.splice(idx, 1);

            resource.addJs('resourcePlaceHolder');
          }

          all = '';
        } else if (!~content.indexOf(opts.resourcePlaceHolder)) {
          all += '\n' + opts.resourcePlaceHolder;
        }

      } else if (rFramework.test(script) && opts.obtainScript) {
        file ? resource.add(file.id, false, false, attrs) : resource.addJs(src, attrs);
      }
    }

    return all;
  });
}

function obtainScriptAndStyle(content, resource, opts, host, includeList) {
  rLinkScript.lastIndex = 0;

  var obtainStyle = ~content.indexOf(opts.stylePlaceHolder) && opts.obtainStyle;
  var obtainScript = ~content.indexOf(opts.scriptPlaceHolder) && opts.obtainScript;

  return content.replace(rLinkScript, function(all, comment, script, attrs, body, link, lattrs, style, sattrs, sbody, ignored) {

    if (comment && ~comment.indexOf(opts.dependenciesInjectPlaceHolder)) {
      host.requires.forEach(function(id) {
        resource.add(id);
      });

      host.asyncs.forEach(function(id) {
        resource.add(id, true);
      });

      includeList.forEach(function(file) {
        resource.add(file.id, true);
      });

      return '';
    } else if (comment) {
      return all;
    } else if (script && !obtainScript) {
      return all;
    } else if ((link || style) && !obtainStyle) {
      return all;
    } else if (ignored) {
      // 如果是 ignored，至少应该把它原来的打包的路径替换掉。
      if (script && !body.trim() && rSrcHref.test(attrs)) {
        var src = RegExp.$2;
        var file = resource.getFileByUrl(src);
        attrs = attrs.replace(rSrcHref, '').replace(/\s+$/, '');

        if (!file) {
          file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), src)));
        }

        if (file) {
          var url = resource.getUri(file.id, true);
          all = script.replace(rSrcHref, ' src="' + url + '"');
        }
      } else if (link && rRefStyle.test(lattrs) && rSrcHref.test(lattrs)) {
        var href = RegExp.$2;
        var file = resource.getFileByUrl(href);
        lattrs = lattrs.replace(rSrcHref, '').replace(/\s+$/, '');

        if (!file) {
          file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), href)));
        }

        if (file) {
          var url = resource.getUri(file.id, true);
          all = link.replace(rSrcHref, ' href="' + url + '"');
        }
      }

      return all;
    }

    if (script && !body.trim() && rSrcHref.test(attrs)) {
      var src = RegExp.$2;
      var file = resource.getFileByUrl(src);
      attrs = attrs.replace(rSrcHref, '').replace(/\s+$/, '');

      if (!file) {
        file = resource.getFileByUrl(fis.util(path.join(path.dirname(host.release), src)));
      }

      file ? resource.add(file.id, false, false, attrs) : resource.addJs(src, attrs);
      all = '';
    } else if (script && !rScriptType.test(attrs) || rScriptType.test(attrs) && ~['text/javascript', 'application/javascript'].indexOf(RegExp.$2.toLowerCase())) {
      resource.addJsEmbed(body, attrs);
      all = '';
    } else if (link && rRefStyle.test(lattrs) && rSrcHref.test(lattrs)) {
      var href = RegExp.$2;
      var file = resource.getFileByUrl(href);
      lattrs = lattrs.replace(rSrcHref, '').replace(/\s+$/, '');

      if (!file) {
{'fraction_non_alphanumeric': 0.2249203821656051, 'fraction_numerical': 0.0027070063694267514, 'mean_word_length': 2.576594533029613, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707864', 'n_tokens_mistral': 4514, 'n_tokens_neox': 4486, 'n_words': 737}