<commit_before>import { join } from "path"
import express, { Router } from "express"
import webpack from "webpack"
import webpackDevMiddleware from "webpack-dev-middleware"
import webpackHotMiddleware from "webpack-hot-middleware"
import historyFallbackMiddleware from "connect-history-api-fallback"
import WebpackErrorNotificationPlugin from "webpack-error-notification"

import opn from "opn"
import debug from "debug"

import collection from "../md-collection-loader/cache.js"
import toUri from "../_utils/to-uri"
import urlAsHtml from "../static/to-html/url-as-html"
import * as pagesActions from "../redux/modules/pages"
// import htmlMetas from "../../_utils/html-metas"

const log = debug("statinamic:builder:server")

// also in md-collection-loader/index.js
const fileExtensionRE = /\.(.*)+$/
let firstRun = true

export default (webpackConfig, options = {}) => {
  options = {
    noDevEntriesTest: /^tests/,
    ...options,
  }
  const { config } = options

  if (!config.baseUrl) {
    throw new Error(
      "You must provide a 'baseUrl' object that contains the following keys:" +
      "'href', 'port', 'hostname'. See https://nodejs.org/api/url.html"
    )
  }

  const devEntries = [
    require.resolve(`webpack-hot-middleware/client`),
  ]

  const devConfig = {
    ...webpackConfig,
    // debug: true,
    // watch: true,
    // colors: true,
    entry: {
      // add devEntries
      ...Object.keys(webpackConfig.entry)
        .reduce(
          (acc, key) => {
            // some entries do not need extra stuff
            acc[key] = key.match(options.noDevEntriesTest) !== null
              ? webpackConfig.entry[key]
              : [
                ...devEntries,
                ...Array.isArray(webpackConfig.entry[key])
                  ? webpackConfig.entry[key]
                  : [ webpackConfig.entry[key] ],
              ]
            return acc
          },
          {}
        ),
    },
    plugins: [
      ...(webpackConfig.plugins || []),
      ...(options.plugins || []),
      new webpack.optimize.OccurenceOrderPlugin(),
      new webpack.HotModuleReplacementPlugin(),
      new webpack.NoErrorsPlugin(),
      new WebpackErrorNotificationPlugin(),
    ],
    eslint: {
      ...webpackConfig.eslint,
      emitWarning: true,
    },
  }

  const server = express()

  // webpack requirements
  const webpackCompiler = webpack(devConfig)

  server.use(webpackDevMiddleware(webpackCompiler, {
    publicPath: webpackConfig.output.publicPath,
    noInfo: !config.verbose,
    ...devConfig.devServer,
  }))

  let entries = []
  webpackCompiler.plugin("done", function(stats) {
    // reset entries
    entries = []
    const namedChunks = stats.compilation.namedChunks
    Object.keys(namedChunks).forEach((chunkName) => {
      entries = [
        ...entries,
        ...namedChunks[chunkName].files.filter(
          (file) => !file.endsWith(".hot-update.js")
        ),
      ]
    })
  })

  // routing for the part we want (starting to the baseUrl pathname)
  const router = Router()
  server.use(config.baseUrl.pathname, router)

  // fallback to index for unknow pages?
  router.use(historyFallbackMiddleware())

  // webpack static ressources
  router.get("*", express.static(webpackConfig.output.path))

  // user static assets
  if (config.assets) {
    server.use(
      config.baseUrl.pathname + config.assets.route,
      express.static(config.assets.path)
    )
  }

  // prerender pages when possible
  const memoryFs = webpackCompiler.outputFileSystem
  router.get("*", (req, res, next) => {
    //                                       ↓ remove first slash
    const uri = toUri(req.originalUrl.slice(1))
    const item = collection.find((item) => item.__url === req.originalUrl)
    if (!item) {
      next()
    }
    else {
      if (
        !req.originalUrl.match(fileExtensionRE) &&
        !req.originalUrl.endsWith("/")
      ) {
        res.redirect(req.originalUrl + "/")
      }

      const relativeUri = item.__dataUrl.replace(config.baseUrl.pathname, "")
      const filepath = join(config.cwd, config.destination, relativeUri)
      const fileContent = memoryFs.readFileSync(filepath)
      log(
        `Using '${ filepath }' to pre-render '${ req.originalUrl }' (${ uri })`
      )
      const data = JSON.parse(fileContent.toString())
      options.store.dispatch({
        type: pagesActions.SET,
        page: uri,
        response: {
          data,
        },
      })

      if (!firstRun) {
        cleanNodeCache(config.cwd)
      }
      firstRun = false

      urlAsHtml(req.originalUrl, {
        exports: options.exports,
        store: options.store,
        // collection: options.collection,
        baseUrl: config.baseUrl,
        assetsFiles: {
          js: entries,
          css: !config.dev,
        },
      })
      .then(
        (html) => {
          res.setHeader("Content-Type", "text/html")
          res.end(html)
        }
      )
      .catch((err) => {
        log(err)
        res.setHeader("Content-Type", "text/plain")
        res.end(err.toString())
      })
    }
  })

  // HMR
  server.use(webpackHotMiddleware(webpackCompiler))

  // THAT'S IT
  server.listen(config.baseUrl.port, config.baseUrl.hostname, (err) => {
    if (err) {
      log(err)

      return
    }
    log(`Dev server started on ${ config.baseUrl.href }`)
    if (config.open) {
      opn(`${ config.baseUrl.href }`)
    }
  })
}
<commit_msg>Fix missing cleanNodeCache import<commit_after>import { join } from "path"
import express, { Router } from "express"
import webpack from "webpack"
import webpackDevMiddleware from "webpack-dev-middleware"
import webpackHotMiddleware from "webpack-hot-middleware"
import historyFallbackMiddleware from "connect-history-api-fallback"
import WebpackErrorNotificationPlugin from "webpack-error-notification"

import opn from "opn"
import debug from "debug"

import collection from "../md-collection-loader/cache.js"
import toUri from "../_utils/to-uri"
import urlAsHtml from "../static/to-html/url-as-html"
import * as pagesActions from "../redux/modules/pages"
// import htmlMetas from "../../_utils/html-metas"
import cleanNodeCache from "../_utils/clean-node-cache"

const log = debug("statinamic:builder:server")

// also in md-collection-loader/index.js
const fileExtensionRE = /\.(.*)+$/
let firstRun = true

export default (webpackConfig, options = {}) => {
  options = {
    noDevEntriesTest: /^tests/,
    ...options,
  }
  const { config } = options

  if (!config.baseUrl) {
    throw new Error(
      "You must provide a 'baseUrl' object that contains the following keys:" +
      "'href', 'port', 'hostname'. See https://nodejs.org/api/url.html"
    )
  }

  const devEntries = [
    require.resolve(`webpack-hot-middleware/client`),
  ]

  const devConfig = {
    ...webpackConfig,
    // debug: true,
    // watch: true,
    // colors: true,
    entry: {
      // add devEntries
      ...Object.keys(webpackConfig.entry)
        .reduce(
          (acc, key) => {
            // some entries do not need extra stuff
            acc[key] = key.match(options.noDevEntriesTest) !== null
              ? webpackConfig.entry[key]
              : [
                ...devEntries,
                ...Array.isArray(webpackConfig.entry[key])
                  ? webpackConfig.entry[key]
                  : [ webpackConfig.entry[key] ],
              ]
            return acc
          },
          {}
        ),
    },
    plugins: [
      ...(webpackConfig.plugins || []),
      ...(options.plugins || []),
      new webpack.optimize.OccurenceOrderPlugin(),
      new webpack.HotModuleReplacementPlugin(),
      new webpack.NoErrorsPlugin(),
      new WebpackErrorNotificationPlugin(),
    ],
    eslint: {
      ...webpackConfig.eslint,
      emitWarning: true,
    },
  }

  const server = express()

  // webpack requirements
  const webpackCompiler = webpack(devConfig)

  server.use(webpackDevMiddleware(webpackCompiler, {
    publicPath: webpackConfig.output.publicPath,
    noInfo: !config.verbose,
    ...devConfig.devServer,
  }))

  let entries = []
  webpackCompiler.plugin("done", function(stats) {
    // reset entries
    entries = []
    const namedChunks = stats.compilation.namedChunks
    Object.keys(namedChunks).forEach((chunkName) => {
      entries = [
        ...entries,
        ...namedChunks[chunkName].files.filter(
          (file) => !file.endsWith(".hot-update.js")
        ),
      ]
    })
  })

  // routing for the part we want (starting to the baseUrl pathname)
  const router = Router()
  server.use(config.baseUrl.pathname, router)

  // fallback to index for unknow pages?
  router.use(historyFallbackMiddleware())

  // webpack static ressources
  router.get("*", express.static(webpackConfig.output.path))

  // user static assets
  if (config.assets) {
    server.use(
      config.baseUrl.pathname + config.assets.route,
      express.static(config.assets.path)
    )
  }

  // prerender pages when possible
  const memoryFs = webpackCompiler.outputFileSystem
  router.get("*", (req, res, next) => {
    //                                       ↓ remove first slash
    const uri = toUri(req.originalUrl.slice(1))
    const item = collection.find((item) => item.__url === req.originalUrl)
    if (!item) {
      next()
    }
    else {
      if (
        !req.originalUrl.match(fileExtensionRE) &&
        !req.originalUrl.endsWith("/")
      ) {
        res.redirect(req.originalUrl + "/")
      }

      const relativeUri = item.__dataUrl.replace(config.baseUrl.pathname, "")
      const filepath = join(config.cwd, config.destination, relativeUri)
      const fileContent = memoryFs.readFileSync(filepath)
      log(
        `Using '${ filepath }' to pre-render '${ req.originalUrl }' (${ uri })`
      )
      const data = JSON.parse(fileContent.toString())
      options.store.dispatch({
        type: pagesActions.SET,
        page: uri,
        response: {
          data,
        },
      })

      if (!firstRun) {
        cleanNodeCache(config.cwd)
      }
      firstRun = false

      urlAsHtml(req.originalUrl, {
        exports: options.exports,
        store: options.store,
        // collection: options.collection,
        baseUrl: config.baseUrl,
        assetsFiles: {
          js: entries,
          css: !config.dev,
        },
      })
      .then(
        (html) => {
          res.setHeader("Content-Type", "text/html")
          res.end(html)
        }
      )
      .catch((err) => {
        log(err)
        res.setHeader("Content-Type", "text/plain")
        res.end(err.toString())
      })
    }
  })

  // HMR
  server.use(webpackHotMiddleware(webpackCompiler))

  // THAT'S IT
  server.listen(config.baseUrl.port, config.baseUrl.hostname, (err) => {
    if (err) {
      log(err)

      return
    }
    log(`Dev server started on ${ config.baseUrl.href }`)
    if (config.open) {
      opn(`${ config.baseUrl.href }`)
    }
  })
}

{'fraction_non_alphanumeric': 0.13774205334307635, 'fraction_numerical': 0.00018268176835951772, 'mean_word_length': 2.2431872037914693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2092398', 'n_tokens_mistral': 3306, 'n_tokens_neox': 3263, 'n_words': 661}