<commit_before>const path = require('path')
const R = require('ramda')
const glob = require('globby')
const frontmatter = require('./util/frontmatter')
const markdown = require('./util/markdown')
const PageUtil = require('./util/page')

const assocPage = (pages, page) =>
  R.assoc(page.id, page, pages)

// turns the list of children from the user provided attributes
// into a list of correctly named childIds
const convertUserProvidedChildrenList = (pageId, attributes = {}) => {
  let { children } = attributes
  if (typeof children !== 'object') return attributes

  const childIds = R.map((childId) =>
    childId.startsWith(`${pageId}/`) ? childId : `${pageId}/${childId}`,
    children
  )

  attributes = R.dissoc('children', attributes)
  attributes = R.assoc('childIds', childIds, attributes)

  return attributes
}

async function readPageFile (filePath) {
  let { attributes, body } = await frontmatter.fromFile(filePath)
  const content = await markdown.fromString(body)
  // prevent empty attributes from being null
  attributes = attributes || {}

  return { attributes, content }
}

async function findPageIds (pagesPath, pathPattern = '**') {
  const pattern = path.resolve(pagesPath, pathPattern, PageUtil.PAGE_FILENAME)
  const pagePaths = await glob(pattern)
  const pageIdFromPageFilePath = R.partial(PageUtil.pageFilePathToPageId, [pagesPath])
  const pageIds = R.map(pageIdFromPageFilePath, pagePaths)

  return pageIds
}

async function findPageFiles (pagesPath, pagePath, childIds = []) {
  const basePath = path.join(pagesPath, pagePath)
  const filesPattern = path.join(basePath, '{,*/}*.*')
  const pageExcludes = ['_{,*/}*.*', `**/${PageUtil.PAGE_FILENAME}`]
  const childExcludes = R.map(id => path.join(id, '**'), childIds)
  const excludes = R.concat(pageExcludes, childExcludes)
  const excludePatterns = R.map((exclude) => '!' + path.join(basePath, exclude), excludes)
  const filePaths = await glob([filesPattern, ...excludePatterns])
<commit_msg>🐛 fix page file exclude globs<commit_after>const path = require('path')
const R = require('ramda')
const glob = require('globby')
const frontmatter = require('./util/frontmatter')
const markdown = require('./util/markdown')
const PageUtil = require('./util/page')

const assocPage = (pages, page) =>
  R.assoc(page.id, page, pages)

// turns the list of children from the user provided attributes
// into a list of correctly named childIds
const convertUserProvidedChildrenList = (pageId, attributes = {}) => {
  let { children } = attributes
  if (typeof children !== 'object') return attributes

  const childIds = R.map((childId) =>
    childId.startsWith(`${pageId}/`) ? childId : `${pageId}/${childId}`,
    children
  )

  attributes = R.dissoc('children', attributes)
  attributes = R.assoc('childIds', childIds, attributes)

  return attributes
}

async function readPageFile (filePath) {
  let { attributes, body } = await frontmatter.fromFile(filePath)
  const content = await markdown.fromString(body)
  // prevent empty attributes from being null
  attributes = attributes || {}

  return { attributes, content }
}

async function findPageIds (pagesPath, pathPattern = '**') {
  const pattern = path.resolve(pagesPath, pathPattern, PageUtil.PAGE_FILENAME)
  const pagePaths = await glob(pattern)
  const pageIdFromPageFilePath = R.partial(PageUtil.pageFilePathToPageId, [pagesPath])
  const pageIds = R.map(pageIdFromPageFilePath, pagePaths)

  return pageIds
}

async function findPageFiles (pagesPath, pagePath, childIds = []) {
  const filesPattern = path.join(pagesPath, pagePath, '{,*/}*.*')
  const pageExcludes = ['**/_{,*/}*.*', `**/${PageUtil.PAGE_FILENAME}`]
  const childExcludes = R.map(id => path.join(id, '**'), childIds)
  const excludes = R.concat(pageExcludes, childExcludes)
  const excludePatterns = R.map((exclude) => '!' + path.join(pagesPath, exclude), excludes)
  const filePaths = await glob([filesPattern, ...excludePatterns])

{'fraction_non_alphanumeric': 0.14086648087154802, 'fraction_numerical': 0.0, 'mean_word_length': 4.927927927927928, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5696675', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1171, 'n_words': 312}