<commit_before>hljs = require 'highlight.js'
marked = require 'marked'
async = require 'async'
path = require 'path'
url = require 'url'
fs = require 'fs'
yaml = require 'js-yaml'

Page = require './page'

is_relative = (uri) ->
  ### returns true if *uri* is relative; otherwise false ###
  (url.parse(uri).protocol == undefined)

parseMarkdownSync = (content, baseUrl) ->
  ### takes markdown *content* and returns html using *baseUrl* for any relative urls
      returns html ###

  marked.inlineLexer.formatUrl = (uri) ->
    if is_relative uri
      return url.resolve baseUrl, uri
    else
      return uri

  tokens = marked.lexer content

  for token in tokens
    switch token.type
      when 'code'
        if token.lang?
          token.text = hljs.highlight(token.lang, token.text).value
        else
          token.text = hljs.highlightAuto(token.text).value
        token.escaped = true

  return marked.parser tokens

class MarkdownPage extends Page

  getLocation: (base) ->
    uri = @getUrl base
    return uri[0..uri.lastIndexOf('/')]

  getHtml: (base) ->
    ### parse @markdown and return html. also resolves any relative urls to absolute ones ###
    @_html ?= parseMarkdownSync @_content, @getLocation(base) # cache html
    return @_html

MarkdownPage.fromFile = (env, filepath, callback) ->
  async.waterfall [
    (callback) ->
      fs.readFile filepath.full, callback
    (buffer, callback) ->
      MarkdownPage.extractMetadata buffer.toString(), callback
    (result, callback) =>
      {markdown, metadata} = result
      page = new this filepath, markdown, metadata
      callback null, page
  ], callback

MarkdownPage.extractMetadata = (content, callback) ->
  parseMetadata = (source, callback) ->
    try
      callback null, yaml.load(source) or {}
    catch error
      callback error
  
  # split metadata and markdown content

  if content[0...3] is '---'
    # "Front Matter"
    result = content.match /^-{3,}\s([\s\S]*?)-{3,}(\s[\s\S]*|\s?)$/
    if result?.length is 3
      metadata = result[1]
      markdown = result[2]
    else
      metadata = ''
      markdown = content
  else
    # old style metadata
    logger.warn 'Deprecation warning: page metadata should be encapsulated by at least three dashes (---)'
    split_idx = content.indexOf '\n\n'
    metadata = content.slice(0, split_idx)
    markdown = content.slice(split_idx + 2)

  async.parallel
    metadata: (callback) ->
      parseMetadata metadata, callback<commit_msg>Drop support for old style metadata<commit_after>hljs = require 'highlight.js'
marked = require 'marked'
async = require 'async'
path = require 'path'
url = require 'url'
fs = require 'fs'
yaml = require 'js-yaml'

Page = require './page'

is_relative = (uri) ->
  ### returns true if *uri* is relative; otherwise false ###
  (url.parse(uri).protocol == undefined)

parseMarkdownSync = (content, baseUrl) ->
  ### takes markdown *content* and returns html using *baseUrl* for any relative urls
      returns html ###

  marked.inlineLexer.formatUrl = (uri) ->
    if is_relative uri
      return url.resolve baseUrl, uri
    else
      return uri

  tokens = marked.lexer content

  for token in tokens
    switch token.type
      when 'code'
        if token.lang?
          token.text = hljs.highlight(token.lang, token.text).value
        else
          token.text = hljs.highlightAuto(token.text).value
        token.escaped = true

  return marked.parser tokens

class MarkdownPage extends Page

  getLocation: (base) ->
    uri = @getUrl base
    return uri[0..uri.lastIndexOf('/')]

  getHtml: (base) ->
    ### parse @markdown and return html. also resolves any relative urls to absolute ones ###
    @_html ?= parseMarkdownSync @_content, @getLocation(base) # cache html
    return @_html

MarkdownPage.fromFile = (env, filepath, callback) ->
  async.waterfall [
    (callback) ->
      fs.readFile filepath.full, callback
    (buffer, callback) ->
      MarkdownPage.extractMetadata buffer.toString(), callback
    (result, callback) =>
      {markdown, metadata} = result
      page = new this filepath, markdown, metadata
      callback null, page
  ], callback

MarkdownPage.extractMetadata = (content, callback) ->
  parseMetadata = (source, callback) ->
    try
      callback null, yaml.load(source) or {}
    catch error
      callback error
  
  # split metadata and markdown content

  if content[0...3] is '---'
    # "Front Matter"
    result = content.match /^-{3,}\s([\s\S]*?)-{3,}(\s[\s\S]*|\s?)$/
    if result?.length is 3
      metadata = result[1]
      markdown = result[2]
    else
      metadata = ''
      markdown = content

  async.parallel
    metadata: (callback) ->
      parseMetadata metadata, callback
{'fraction_non_alphanumeric': 0.12063492063492064, 'fraction_numerical': 0.0038095238095238095, 'mean_word_length': 2.8642681929681113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3496430', 'n_tokens_mistral': 1444, 'n_tokens_neox': 1399, 'n_words': 426}