<commit_before>import glob from 'glob'
import path from 'path'
import parsePath from 'parse-filepath'
import slash from 'slash'
import fs from 'fs'
import frontMatter from 'front-matter'
import htmlFrontMatter from 'html-frontmatter'
import objectAssign from 'object-assign'
const debug = require('debug')('gatsby:glob')
let rewritePath
try {
  const gatsbyNodeConfig = path.resolve(process.cwd(), './gatsby-node.js')
  const nodeConfig = require(gatsbyNodeConfig)
  rewritePath = nodeConfig.rewritePath
} catch (e) {
  // Ignore
}

module.exports = (directory, callback) => {
  const pagesData = []

  // Make this list easy to modify through the config?
  // Or just keep adding extensions...?
  const fileTypesToGlob = [
    'coffee',
    'cjsx',
    'jsx',
    'js',
    'md',
    'html',
    'json',
    'yaml',
    'toml',
  ]
  const fileGlobQuery = fileTypesToGlob.map((type) => `*.${type}`)
  const joinedFileQuery = fileGlobQuery.join('|')
  const globQuery = `${directory}/pages/**/?(${joinedFileQuery})`
  glob(globQuery, null, (err, pages) => {
    if (err) { return callback(err) }

    pages.forEach((page) => {
      const pageData = {}
      pageData.file = {}

      pageData.file = parsePath(path.relative(`${directory}/pages`, page))
      const parsed = pageData.file

      pageData.file.ext = parsed.extname.slice(1)
      const ext = pageData.file.ext

      // Determine require path
      pageData.requirePath = slash(path.relative(`${directory}/pages`, page))

      // Make sure slashes on parsed.dirname are correct for Windows
      parsed.dirname = slash(parsed.dirname)

      // Load data for each file type.
      // TODO use webpack-require to ensure data loaded
      // here (in node context) is consistent with what's loaded
      // in the browser.
      let data
      if (ext === 'md') {
        const rawData = frontMatter(fs.readFileSync(page, 'utf-8'))
        data = objectAssign({}, rawData.attributes)
        pageData.data = data
      } else if (ext === 'html') {
        const html = fs.readFileSync(page, 'utf-8')
        data = objectAssign({}, htmlFrontMatter(html), { body: html })
        pageData.data = data
      } else {
        data = {}
      }

      // Determine path for page (unless it's a file that starts with an
      // underscore as these don't become pages).
      if (!(parsed.name.slice(0, 1) === '_')) {
        if (data.path) {
          // Path was hardcoded.
          pageData.path = data.path
        } else if (rewritePath) {
          pageData.path = rewritePath(parsed, pageData)
        }

        // If the above didn't set a path, set our own.<commit_msg>Adding a check to verify path strings in front matter are prepended with a forward slash<commit_after>import glob from 'glob'
import path from 'path'
import parsePath from 'parse-filepath'
import slash from 'slash'
import fs from 'fs'
import frontMatter from 'front-matter'
import htmlFrontMatter from 'html-frontmatter'
import objectAssign from 'object-assign'
import invariant from 'invariant'
const debug = require('debug')('gatsby:glob')
let rewritePath
try {
  const gatsbyNodeConfig = path.resolve(process.cwd(), './gatsby-node.js')
  const nodeConfig = require(gatsbyNodeConfig)
  rewritePath = nodeConfig.rewritePath
} catch (e) {
  // Ignore
}

module.exports = (directory, callback) => {
  const pagesData = []

  // Make this list easy to modify through the config?
  // Or just keep adding extensions...?
  const fileTypesToGlob = [
    'coffee',
    'cjsx',
    'jsx',
    'js',
    'md',
    'html',
    'json',
    'yaml',
    'toml',
  ]
  const fileGlobQuery = fileTypesToGlob.map((type) => `*.${type}`)
  const joinedFileQuery = fileGlobQuery.join('|')
  const globQuery = `${directory}/pages/**/?(${joinedFileQuery})`
  glob(globQuery, null, (err, pages) => {
    if (err) { return callback(err) }

    pages.forEach((page) => {
      const pageData = {}
      pageData.file = {}

      pageData.file = parsePath(path.relative(`${directory}/pages`, page))
      const parsed = pageData.file

      pageData.file.ext = parsed.extname.slice(1)
      const ext = pageData.file.ext

      // Determine require path
      pageData.requirePath = slash(path.relative(`${directory}/pages`, page))

      // Make sure slashes on parsed.dirname are correct for Windows
      parsed.dirname = slash(parsed.dirname)

      // Load data for each file type.
      // TODO use webpack-require to ensure data loaded
      // here (in node context) is consistent with what's loaded
      // in the browser.
      let data
      if (ext === 'md') {
        const rawData = frontMatter(fs.readFileSync(page, 'utf-8'))
        data = objectAssign({}, rawData.attributes)
        pageData.data = data
      } else if (ext === 'html') {
        const html = fs.readFileSync(page, 'utf-8')
        data = objectAssign({}, htmlFrontMatter(html), { body: html })
        pageData.data = data
      } else {
        data = {}
      }

      // Determine path for page (unless it's a file that starts with an
      // underscore as these don't become pages).
      if (!(parsed.name.slice(0, 1) === '_')) {
        if (data.path) {
          // Path was hardcoded.
          const pathInvariantMessage = `
          You're setting path to ${data.path} for ${parsed.dirname},
          but it should be /${data.path}
          `
          invariant((data.path.charAt(0) === '/'), pathInvariantMessage)
          pageData.path = data.path
        } else if (rewritePath) {
          pageData.path = rewritePath(parsed, pageData)
        }

        // If the above didn't set a path, set our own.
{'fraction_non_alphanumeric': 0.1328, 'fraction_numerical': 0.0019555555555555554, 'mean_word_length': 2.6627604166666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4787896', 'n_tokens_mistral': 1737, 'n_tokens_neox': 1688, 'n_words': 494}