<commit_before>import fs from "fs"
import path from "path"
import crypto from "crypto"
import denodeify from "denodeify"
import fse from "fs-extra"

import postcss from "postcss"
import postcssSmartImport from "postcss-smart-import"
import postcssParserSugarss from "sugarss"
import postcssParserScss from "postcss-scss"

var copyAsync = denodeify(fse.copy)
var writeAsync = denodeify(fse.outputFile)

function isAssetFile(id) {
  var fileExt = path.extname(id).slice(1)
  return !(fileExt === "" || (/^(json|jsx|js|es|es5|es6)$/).exec(fileExt))
}

const styleExtensions =
{
  ".css": null,
  ".sss": postcssParserSugarss,
  ".scss": postcssParserScss
}

const postcssPlugins = [
  postcssSmartImport()
]

function processStyle(code, id, dest)
{
  var parser = styleExtensions[path.extname(id)]
  return postcss(postcssPlugins)
    .process(code,
      {
        from: id,
        to: dest,
        parser: parser,
        extensions: ['.css', '.sss', '.scss']
      })
    .then((result) => {
      return writeAsync(dest, result)
    })
    .catch(function(err) {
      console.error(err)
    })
}

const staticAST = {
  type: "Program",
  sourceType: "module",
  start: 0,
  end: null,
  body: []
}


export default function(outputFolder)
{
  return {
    name: "file-loader",

    transform: function(code, id)
    {
      if (!isAssetFile(id))
        return null

      return {
        code: `export default "FIXME-PROTECT-IMPORT"`,
        map: { mappings: "" }
      }
    },

    load: function(id)
    {
      if (!isAssetFile(id))
        return null

      const input = fs.createReadStream(id)
      const hash = crypto.createHash("sha256")

      var fileData = ""

      return new Promise((resolve, reject) =>
      {
        input.on("readable", () =>
        {
          var data = input.read()
          if (data)
          {
            fileData += data
            hash.update(data);
          }
          else
          {
            var fileHasher = crypto.createHash("sha1")
            var fileSource = id
            var fileExt = path.extname(id)
            var destExt = fileExt in styleExtensions ? ".css" : fileExt
            var fileHash = hash.digest("hex").slice(0, 8)
            var idDest = path.basename(id, fileExt) + "-" + fileHash + destExt
            var fileDest = path.join(outputFolder, idDest)

            if (fileExt in styleExtensions)
            {
              console.log("Processing style:", id)

              return processStyle(fileData, fileSource, fileDest).then(function()
              {
                console.log("Processed:", fileSource, "=>", fileDest)

                resolve({
                  ast : staticAST,
                  code: `import h${fileHash} from "./${idDest}"; export default h${fileHash};`,
                  map: { mappings: "" }
                })
              })
            }
            else
            {
              console.log("Processing asset:", id)

              return copyAsync(fileSource, fileDest).then(function()
              {
                console.log("Copied:", fileSource, "=>", fileDest)

                resolve({
                  ast : staticAST,
                  code: `import h${fileHash} from "./${idDest}"; export default h${fileHash};`,
                  map: { mappings: "" }
                })
              })
            }
          }
        })
      })
    }
  }
}
<commit_msg>Added marking external solution. Unfortunately does not seem to work correctly in current rollup.<commit_after>import fs from "fs"
import path from "path"
import crypto from "crypto"
import denodeify from "denodeify"
import fse from "fs-extra"

import postcss from "postcss"
import postcssSmartImport from "postcss-smart-import"
import postcssParserSugarss from "sugarss"
import postcssParserScss from "postcss-scss"

var copyAsync = denodeify(fse.copy)
var writeAsync = denodeify(fse.outputFile)

function isAssetFile(id) {
  var fileExt = path.extname(id).slice(1)
  return !(fileExt === "" || (/^(json|jsx|js|es|es5|es6)$/).exec(fileExt))
}

const styleExtensions =
{
  ".css": null,
  ".sss": postcssParserSugarss,
  ".scss": postcssParserScss
}

const postcssPlugins = [
  postcssSmartImport()
]

function processStyle(code, id, dest)
{
  var parser = styleExtensions[path.extname(id)]
  return postcss(postcssPlugins)
    .process(code,
      {
        from: id,
        to: dest,
        parser: parser,
        extensions: ['.css', '.sss', '.scss']
      })
    .then((result) => {
      return writeAsync(dest, result)
    })
    .catch(function(err) {
      console.error(err)
    })
}

const staticAST = {
  type: "Program",
  sourceType: "module",
  start: 0,
  end: null,
  body: []
}

const externalIds = {}


export default function(outputFolder)
{
  return {
    name: "file-loader",

    resolveId: function(importee, importer) {
      console.log("RR: Testing:", importee)
      if (importee in externalIds) {
        console.log("RR: Marking external!")

        // This does not seem to work!

        // "returning any other falsy value signals that importee should be treated as an external module
        // and not included in the bundle." -- https://github.com/rollup/rollup/wiki/Plugins#creating-plugins
        return false
      }

      console.log("RR: Delegating...")
    },


    transformxx: function(code, id)
    {
      if (!isAssetFile(id))
        return null

      return {
        code: `export default "FIXME-PROTECT-IMPORT"`,
        map: { mappings: "" }
      }
    },

    load: function(id)
    {
      if (!isAssetFile(id))
        return null

      const input = fs.createReadStream(id)
      const hash = crypto.createHash("sha256")

      var fileData = ""

      return new Promise((resolve, reject) =>
      {
        input.on("readable", () =>
        {
          var data = input.read()
          if (data)
          {
            fileData += data
            hash.update(data);
          }
          else
          {
            var fileHasher = crypto.createHash("sha1")
            var fileSource = id
            var fileExt = path.extname(id)
            var destExt = fileExt in styleExtensions ? ".css" : fileExt
            var fileHash = hash.digest("hex").slice(0, 8)
            var idDest = path.basename(id, fileExt) + "-" + fileHash + destExt
            var fileDest = path.join(outputFolder, idDest)

            externalIds["./" + idDest] = true

            if (fileExt in styleExtensions)
            {
              console.log("Processing style:", id)

              return processStyle(fileData, fileSource, fileDest).then(function()
              {
                console.log("Processed:", fileSource, "=>", fileDest)

                resolve({
                  ast : staticAST,
                  code: `import h${fileHash} from "./${idDest}"; export default h${fileHash};`,
                  map: { mappings: "" }
                })
              })
            }
            else
            {
              console.log("Processing asset:", id)

              return copyAsync(fileSource, fileDest).then(function()
              {
                console.log("Copied:", fileSource, "=>", fileDest)

                resolve({
                  ast : staticAST,
                  code: `import h${fileHash} from "./${idDest}"; export default h${fileHash};`,
                  map: { mappings: "" }
                })
              })
            }
          }
        })
      })
    }
  }
}

{'fraction_non_alphanumeric': 0.12777332264100508, 'fraction_numerical': 0.002673082063619353, 'mean_word_length': 1.8825115562403698, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4687772', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2174, 'n_words': 515}