<commit_before>const fs = require('fs')
const URL = require('url')
const { promisify } = require('util')
const { types, extensions } = require('mime-types')
const readdir = promisify(fs.readdir)

// A ResourceMapper maintains the mapping between HTTP URLs and server filenames,
// following the principles of the “sweet spot” discussed in
// https://www.w3.org/DesignIssues/HTTPFilenameMapping.html
class ResourceMapper {
  constructor ({
    rootUrl,
    rootPath,
    includeHost = false,
    defaultContentType = 'application/octet-stream',
    indexFilename = 'index',
    overrideTypes = { acl: 'text/turtle', meta: 'text/turtle' }
  }) {
    this._rootUrl = this._removeTrailingSlash(rootUrl)
    this._rootPath = this._removeTrailingSlash(rootPath)
    this._includeHost = includeHost
    this._readdir = readdir
    this._defaultContentType = defaultContentType
    this._indexFilename = indexFilename
    this._types = { ...types, ...overrideTypes }

    // If the host needs to be replaced on every call, pre-split the root URL
    if (includeHost) {
      const { protocol, port, pathname } = URL.parse(rootUrl)
      this._protocol = protocol
      this._port = port === null ? '' : `:${port}`
      this._rootUrl = this._removeTrailingSlash(pathname)
    }
  }

  get rootPath () {
    return this._rootPath
  }

  // Maps the request for a given resource and representation format to a server file
  // When the URL ends with a '/', then files with the prefix 'index.' will be matched,
  // such as 'index.html' and 'index.ttl'.
  async mapUrlToFile ({ url, contentType, createIfNotExists }) {
    let fullPath = this._getFullPath(url)
    let isIndex = fullPath.endsWith('/')
    let path

    // Append index filename if the URL ends with a '/'
    if (isIndex) {
      fullPath += this._indexFilename
    }

    // Create the path for a new file
    if (createIfNotExists) {
      path = fullPath
      // If the extension is not correct for the content type, append the correct extension
      if (this._getContentTypeByExtension(path) !== contentType) {
        // Append a '$', unless we map for the index
        if (!isIndex) {
          path += '$'
        }
        path += contentType in extensions ? `.${extensions[contentType][0]}` : '.unknown'
      }
    // Determine the path of an existing file
    } else {
      // Read all files in the corresponding folder
      const filename = fullPath.substr(fullPath.lastIndexOf('/') + 1)
      const folder = fullPath.substr(0, fullPath.length - filename.length)
      const files = await this._readdir(folder)

      // Find a file with the same name (minus the dollar extension)
      let match = files.find(f => this._removeDollarExtension(f) === filename ||
        (isIndex && f.startsWith(this._indexFilename + '.')))
      if (!match) {
        // Error if no match was found,
        // unless the URL ends with a '/',
        // in that case we fallback to the folder itself.<commit_msg>Document removal of $ with indexes in ResourceMapper<commit_after>const fs = require('fs')
const URL = require('url')
const { promisify } = require('util')
const { types, extensions } = require('mime-types')
const readdir = promisify(fs.readdir)

// A ResourceMapper maintains the mapping between HTTP URLs and server filenames,
// following the principles of the “sweet spot” discussed in
// https://www.w3.org/DesignIssues/HTTPFilenameMapping.html
class ResourceMapper {
  constructor ({
    rootUrl,
    rootPath,
    includeHost = false,
    defaultContentType = 'application/octet-stream',
    indexFilename = 'index',
    overrideTypes = { acl: 'text/turtle', meta: 'text/turtle' }
  }) {
    this._rootUrl = this._removeTrailingSlash(rootUrl)
    this._rootPath = this._removeTrailingSlash(rootPath)
    this._includeHost = includeHost
    this._readdir = readdir
    this._defaultContentType = defaultContentType
    this._indexFilename = indexFilename
    this._types = { ...types, ...overrideTypes }

    // If the host needs to be replaced on every call, pre-split the root URL
    if (includeHost) {
      const { protocol, port, pathname } = URL.parse(rootUrl)
      this._protocol = protocol
      this._port = port === null ? '' : `:${port}`
      this._rootUrl = this._removeTrailingSlash(pathname)
    }
  }

  get rootPath () {
    return this._rootPath
  }

  // Maps the request for a given resource and representation format to a server file
  // When the URL ends with a '/', then files with the prefix 'index.' will be matched,
  // such as 'index.html' and 'index.ttl'.
  async mapUrlToFile ({ url, contentType, createIfNotExists }) {
    let fullPath = this._getFullPath(url)
    let isIndex = fullPath.endsWith('/')
    let path

    // Append index filename if the URL ends with a '/'
    if (isIndex) {
      fullPath += this._indexFilename
    }

    // Create the path for a new file
    if (createIfNotExists) {
      path = fullPath
      // If the extension is not correct for the content type, append the correct extension
      if (this._getContentTypeByExtension(path) !== contentType) {
        // Append a '$', unless we map for the index
        // Because the index must _always_ have a proper extension when it is being created
        if (!isIndex) {
          path += '$'
        }
        path += contentType in extensions ? `.${extensions[contentType][0]}` : '.unknown'
      }
    // Determine the path of an existing file
    } else {
      // Read all files in the corresponding folder
      const filename = fullPath.substr(fullPath.lastIndexOf('/') + 1)
      const folder = fullPath.substr(0, fullPath.length - filename.length)
      const files = await this._readdir(folder)

      // Find a file with the same name (minus the dollar extension)
      let match = files.find(f => this._removeDollarExtension(f) === filename ||
        (isIndex && f.startsWith(this._indexFilename + '.')))
      if (!match) {
        // Error if no match was found,
        // unless the URL ends with a '/',
        // in that case we fallback to the folder itself.
{'fraction_non_alphanumeric': 0.11023232822540781, 'fraction_numerical': 0.0013181743285549513, 'mean_word_length': 2.8393421884882986, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513025', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1696, 'n_words': 569}