<commit_before>#!/usr/bin/env node

var fs = require('fs')
var glob = require('glob')
var path = require('path')

// We are parsing the doc files for links
// to other pages within the docs (relative links,
// rather than external).
//
// We're looking for these patterns:
// - [words](../abc/abc.md)
// - [words](../abc/abc)
// - [words](abc)
// - [words](abc.md)
// - [words](#abc-efg)
// - [words](../abc/abc.html)
// - [words](/abc/abc.html)
// - [words](_abc/abc/abc.html)

module.exports = function fixLinks (dir, callback) {
  var filesWithLinks = 0
  var filesToCheck = 0
  var filesProcessed = 0
  var done = false

  glob('**/*.md', {cwd: dir}, function globbing (error, files) {
    if (error) return callback(error)
    if (files.length === 0) return callback(new Error('Globbed no files.'))
    filesToCheck = files.length
    files.forEach(function (file, i) {
      file = path.join(dir, file)
      findLinks(file, i)
    })
  })

  function findLinks (file, i) {
    var urlRegex = /\[.+\]\((?!http)(.+)\)/g
    var content = fs.readFileSync(file).toString()
    var links = content.match(urlRegex)
    if (links) {
      filesWithLinks++
      fixInternalLinks(file, content, links)
    }
    if (filesToCheck === i + 1) done = true
  }

  function fixInternalLinks (file, content, links) {
    var linksCounter = 0

    var newLinks = links.map(function fixLinks (link, i) {
      linksCounter++
      if (link.match(/\.md\)/ig)) {
        var newLink = link.replace(/\.md\)/ig, ')')
        if (newLink.indexOf('../') > -1) {
          return newLink.replace(/\(\.\.\//ig, '(../../')
        } else if (newLink.indexOf('./') > -1) {
          return newLink.replace(/\(\.\//ig, '(../')
        } else if (newLink.indexOf('/') < 0) {
          return newLink.replace('](', '](../')
        }
        return newLink
      }
      return link
    })
    if (newLinks.length === links.length) {
      replaceLinksInContent(file, content, links, newLinks)
    }
  }

  function replaceLinksInContent (file, content, oldLinks, newLinks) {
    oldLinks.forEach(function (oldLink, i) {
      content = content.replace(oldLink, newLinks[i])
    })
    writeLinkstoFile(file, content)
  }

  function writeLinkstoFile (file, content) {
    fs.writeFile(file, content, function done (error) {
      if (error) return callback(error)
      filesProcessed++
      if (filesProcessed === filesWithLinks && done) {
        callback(null, 'done')
      }
    })
  }
}<commit_msg>Build absolute urls<commit_after>#!/usr/bin/env node

var fs = require('fs')
var glob = require('glob')
var path = require('path')

// We are parsing the doc files for links
// to other pages within the docs (relative links,
// rather than external).
//
// We're looking for these patterns:
// - [words](../abc/abc.md)
// - [words](../abc/abc)
// - [words](abc)
// - [words](abc.md)
// - [words](#abc-efg)
// - [words](../abc/abc.html)
// - [words](/abc/abc.html)
// - [words](_abc/abc/abc.html)

module.exports = function fixLinks (dir, version, callback) {
  var filesWithLinks = 0
  var filesToCheck = 0
  var filesProcessed = 0
  var done = false

  glob('**/*.md', {cwd: dir}, function globbing (error, files) {
    if (error) return callback(error)
    if (files.length === 0) return callback(new Error('Globbed no files.'))
    filesToCheck = files.length
    files.forEach(function (file, i) {
      file = path.join(dir, file)
      findLinks(file, i)
    })
    if (filesWithLinks === 0) return callback(new Error('Found no relative links in files.'))
  })

  function findLinks (file, i) {
    var urlRegex = /\[.+\]\((?!http)(.+)\)/g
    var content = fs.readFileSync(file).toString()
    var links = content.match(urlRegex)
    if (links) {
      filesWithLinks++
      fixInternalLinks(file, content, links)
    }
    if (filesToCheck === (i + 1)) done = true
  }

  function fixInternalLinks (file, content, links) {
    var baseUrl = 'electron.atom.io/docs/' + version

    var newLinks = links.map(function fixLinks (link, i) {
      var parts = link.split('](')
      var href = parts[1]
      href = href.slice(0, href.length - 1)
      var bracket = parts[0] + ']'

      if (href.match(/\.md/ig)) {
        var newLink = href.replace(/\.md/ig, '')
        if (newLink.indexOf('../') > -1) {
          newLink = 'http://' + path.join(baseUrl, newLink)
          var newMdLink = bracket + '(' + newLink + ')'
          return newMdLink
        } else if (newLink.indexOf('./') > -1) {
          var fileDir = getFileDir(file)
          newLink = 'http://' + path.join(baseUrl, fileDir, newLink)
          newMdLink = bracket + '(' + newLink + ')'
          return newMdLink
        } else if (newLink.indexOf('/') < 0) {
          fileDir = getFileDir(file)
          newLink = 'http://' + path.join(baseUrl, fileDir, newLink)
          newMdLink = bracket + '(' + newLink + ')'
          return newMdLink
        }
        return newLink
      }
      return link
    })
    replaceLinksInContent(file, content, links, newLinks)
  }

  function getFileDir (path) {
    var array = path.split('/')
    return array[array.length - 2]
  }

  function replaceLinksInContent (file, content, oldLinks, newLinks) {
    oldLinks.forEach(function (oldLink, i) {
      content = content.replace(oldLink, newLinks[i])
    })
    writeLinkstoFile(file, content)
  }

  function writeLinkstoFile (file, content) {
    fs.writeFile(file, content, function (error) {
      if (error) return callback(error)
      filesProcessed++
      if ((filesProcessed === filesWithLinks) && done) {
        callback(null, 'done')
      }
    })
  }
}
{'fraction_non_alphanumeric': 0.1708140774973338, 'fraction_numerical': 0.004088162104514753, 'mean_word_length': 2.495031055900621, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3693025', 'n_tokens_mistral': 1884, 'n_tokens_neox': 1833, 'n_words': 453}