<commit_before>const {filter, mapObjIndexed, map, values, flatten, pipe} = require("ramda")
const addFilenameToLinks = require("./add-filename-to-links")
const getFileContents = require("./get-file-contents")
const pickHtmlFiles = require("./pick-html-files")
const shouldCheckLink = require("./should-check-link")
const normalizeFiles = require("./normalize-files")
const isLinkBroken = require("./is-link-broken")
const extractLinks = require("./extract-links")
const path = require("path")

function fileExists(files, filename) {
  // Remove leading slash before checking to match the Metalsmith files format
  if (filename.charAt(0) === "/") {
    filename = filename.slice(1)
  }

  // True if it exists in the files object
  if (filename in files) {
    return true
  }

  // Fallback for Windows paths
  const winPath = filename.split("/").join(path.sep)
  return winPath in files
}

module.exports = (options) => {

  if (options == null) { options = {} }
  if (options === true) { options = {} } // Allow CLI to specify true
  if (options.warn == null) { options.warn = false }
  if (options.checkLinks == null) { options.checkLinks = true }
  if (options.checkImages == null) { options.checkImages = true }
  if (options.allowRegex == null) { options.allowRegex = null }
  if (options.allowAnchors == null) { options.allowAnchors = true }
  if (options.baseURL == null) { options.baseURL = null }

  return (files) => {
    // const normalized = normalizeFiles(files)
    const normalized = files

    function handleLinks(links) {
      links.forEach((link) => {
        const broken = isLinkBroken({
          files,
          fileExists,
          link,
          options,
        })

        if (broken) {
          if (options.warn) {
            console.log(`Warning: Link is broken: ${link.description}, in file: ${link.filename}`)
          } else {
            throw new Error(`Link is broken: ${link.description}, in file: ${link.filename}`)
          }
        }
      })
    }

    pipe(
      pickHtmlFiles,
      map(getFileContents),
      map(extractLinks),
      mapObjIndexed(addFilenameToLinks),
      values,
      flatten,
      filter(shouldCheckLink(options)),
      handleLinks
    )(normalized)
  }
}<commit_msg>Refactor structure of index file<commit_after>const {assoc, filter, mapObjIndexed, map, values, flatten, pipe} = require("ramda")
const addFilenameToLinks = require("./add-filename-to-links")
const getFileContents = require("./get-file-contents")
const pickHtmlFiles = require("./pick-html-files")
const shouldCheckLink = require("./should-check-link")
const normalizeFiles = require("./normalize-files")
const isLinkBroken = require("./is-link-broken")
const extractLinks = require("./extract-links")
const path = require("path")

function fileExists(files, filename) {
  // Remove leading slash before checking to match the Metalsmith files format
  if (filename.charAt(0) === "/") {
    filename = filename.slice(1)
  }

  // True if it exists in the files object
  if (filename in files) {
    return true
  }

  // Fallback for Windows paths
  const winPath = filename.split("/").join(path.sep)
  return winPath in files
}

module.exports = (options) => {

  if (options == null) { options = {} }
  if (options === true) { options = {} } // Allow CLI to specify true
  if (options.warn == null) { options.warn = false }
  if (options.checkLinks == null) { options.checkLinks = true }
  if (options.checkImages == null) { options.checkImages = true }
  if (options.allowRegex == null) { options.allowRegex = null }
  if (options.allowAnchors == null) { options.allowAnchors = true }
  if (options.baseURL == null) { options.baseURL = null }

  return (files) => {
    // const normalized = normalizeFiles(files)
    const normalized = files

    pipe(
      pickHtmlFiles,
      map(getFileContents),
      map(extractLinks),
      mapObjIndexed(addFilenameToLinks),
      values,
      flatten,
      filter(shouldCheckLink(options)),
      map(addBrokenStatus(files, fileExists, options)),
      handleLinks(options)
    )(normalized)
  }
}

function addBrokenStatus(files, fileExists, options) {
  return (link) => {
    const broken = isLinkBroken({
      files,
      fileExists,
      link,
      options,
    })

    return assoc("broken", broken, link)
  }
}

function handleLinks(options) {
  return (links) => {
    links.forEach((link) => {
      if (link.broken) {
        if (options.warn) {
          console.log(`Warning: Link is broken: ${link.description}, in file: ${link.filename}`)
        } else {
          throw new Error(`Link is broken: ${link.description}, in file: ${link.filename}`)
        }
      }
    })
  }
}
{'fraction_non_alphanumeric': 0.13789428815004262, 'fraction_numerical': 0.0008525149190110827, 'mean_word_length': 3.0632034632034633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '172183', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1373, 'n_words': 360}