<commit_before># Generate the documentation for a source file by reading it in, splitting it
# up into comment/code sections, highlighting them for the appropriate language,
# and merging them into an HTML template.
generateDocumentation = (source, context, callback) ->
  fs.readFile source, "utf-8", (error, code) ->
    throw error if error
    sections = parse source, code
    highlight source, sections, ->
      generate_source_html source, context, sections
      callback()

# Given a string of source code, parse out each comment and the code that
# follows it, and create an individual **section** for it.
# Sections take the form:
#
#     {
#       docs_text: ...
#       docs_html: ...
#       code_text: ...
#       code_html: ...
#     }
#
parse = (source, code) ->
  lines    = code.split '\n'
  sections = []
  language = get_language source
  has_code = docs_text = code_text = ''

  in_multi = false
  multi_accum = ""

  save = (docs, code) ->
    sections.push docs_text: docs, code_text: code
  for line in lines
    if line.match(language.multi_start_matcher) or in_multi

      if has_code
        save docs_text, code_text
        has_code = docs_text = code_text = ''

      # Found the start of a multiline comment line, set in_multi to true
      # and begin accumulating lines untime we reach a line that finishes
      # the multiline comment
      in_multi = true
      multi_accum += line + '\n'

      # If we reached the end of a multiline comment, template the result
      # and set in_multi to false, reset multi_accum
      if line.match(language.multi_end_matcher)
        in_multi = false
        try
          parsed = dox.parseComments( multi_accum )[0]
          docs_text += dox_template(parsed)
        catch error
          console.log "Error parsing comments with Dox: #{error}"
          docs_text = multi_accum
        multi_accum = ''

    else if line.match(language.comment_matcher) and not line.match(language.comment_filter)
      if has_code
        save docs_text, code_text
        has_code = docs_text = code_text = ''
      docs_text += line.replace(language.comment_matcher, '') + '\n'
    else
      has_code = yes
      code_text += line + '\n'
  save docs_text, code_text
  sections

# Highlights a single chunk of CoffeeScript code, using **Pygments** over stdio,
# and runs the text of its corresponding comment through **Markdown**, using
# [Showdown.js](http://attacklab.net/showdown/).
#
# We process the entire file in a single call to Pygments by inserting little
# marker comments between each section and then splitting the result string
# wherever our markers occur.
highlight = (source, sections, callback) ->
  for section, i in sections
    section.code_html = highlight_start + section.code_text + highlight_end
    section.docs_html = showdown.makeHtml section.docs_text
  callback()
   
# Once all of the code is finished highlighting, we can generate the HTML file
# and write out the documentation. Pass the completed sections into the template
# found in `resources/docco.jst`
generate_source_html = (source, context, sections) ->
  title = path.basename source
  dest  = destination source, context
  html  = docco_template {
    title: title, file_path: source, sections: sections, context: context, path: path, relative_base: relative_base
  }

  console.log "docco: #{source} -> #{dest}"
  write_file(dest, html)

generate_readme = (context, sources, package_json) ->
  title = "README"
  dest = "#{context.config.output_dir}/index.html"
  source = "README.md"

  # README.md template to be use to generate the main README file
  readme_template  = jade.compile fs.readFileSync(__dirname + '/../resources/readme.jade').toString(), { filename: __dirname + '/../resources/readme.jade' }
  readme_path = "#{process.cwd()}/#{source}"
  content_index_path = "#{process.cwd()}/#{context.config.content_dir}/content_index.md"
  
  # generate the content index if it exists under the content sources
  if file_exists(content_index_path)
    content_index = parse_markdown context, content_index_path
  else
    content_index = ""
 
  # parse the markdown the the readme 
  content = parse_markdown(context, readme_path) || "There is no #{source} for this project yet :( "
  
  # run cloc 
  cloc sources.join(" "), (code_stats) ->

    html = readme_template {
      title: title
      context: context
      content: content
      content_index: content_index
      file_path: source
      path: path
      relative_base: relative_base
      package_json: package_json
      code_stats: code_stats
      gravatar: gravatar
    }
    
    console.log "docco: #{source} -> #{dest}"
    write_file(dest, html)

generate_content = (context, dir) ->
  walker = walk.walk(dir, { followLinks: false })
  walker.on 'file', (root, fileStats, next) ->
    # only match files that end in *.md
    if fileStats.name.match(new RegExp ".md$")
      src = "#{root}/#{fileStats.name}"
      dest  = destination(src.replace(context.config.content_dir, ""), context)
      console.log "markdown: #{src} --> #{dest}"
      html = parse_markdown context, src
      html = content_template {
        title: fileStats.name, context: context, content: html, file_path: fileStats.name, path: path, relative_base: relative_base
      }
      write_file dest, html
    next()

# Write a file to the filesystem
write_file = (dest, contents) ->

    target_dir = path.dirname(dest)
    write_func = ->
      fs.writeFile dest, contents, (err) -> throw err if err

    fs.stat target_dir, (err, stats) ->
      throw err if err and err.code != 'ENOENT'

      return write_func() unless err

      if err
        exec "mkdir -p #{target_dir}", (err) ->
          throw err if err
          write_func()


# Parse a markdown file and return the HTML 
parse_markdown = (context, src) ->
  markdown = fs.readFileSync(src).toString()
  return showdown.makeHtml markdown

cloc = (paths, callback) ->
  exec "#{__dirname}/../vendor/cloc.pl --quiet --read-lang-def=#{__dirname}/../resources/cloc_definitions.txt #{paths}", (err, stdout) ->
    console.log "Calculating project stats failed #{err}" if err
    callback stdout

#### Helpers & Setup

# Require our external dependencies, including **Showdown.js**
# (the JavaScript implementation of Markdown).
fs       = require 'fs'
path     = require 'path'
showdown = require('./../vendor/showdown').Showdown
jade     = require 'jade'
dox      = require 'dox'
gravatar = require 'gravatar'
_        = require 'underscore'
walk     = require 'walk'
{spawn, exec} = require 'child_process'

# A list of the languages that Docco supports, mapping the file extension to
# the name of the Pygments lexer and the symbol that indicates a comment. To
# add another language to Docco's repertoire, add it here.
languages =
  '.css':
    name: 'css', symbol: '//', multi_start: "/*", multi_end: "*/"
  '.scss':
    name: 'scss', symbol: '//', multi_start: "/*", multi_end: "*/"
  '.less':
    name: 'less', symbol: '//', multi_start: "/*", multi_end: "*/"
<commit_msg>Refactor and remove irrelevant and obsolete comments<commit_after>#### Helpers & Setup

# Require our external dependencies, including **Showdown.js**
# (the JavaScript implementation of Markdown).
fs       = require 'fs'
path     = require 'path'
showdown = require('./../vendor/showdown').Showdown
jade     = require 'jade'
dox      = require 'dox'
gravatar = require 'gravatar'
_        = require 'underscore'
walk     = require 'walk'
{spawn, exec} = require 'child_process'

# Generate the documentation for a source file by reading it in, splitting it
# up into comment/code sections, and merging them into an HTML template.
generateDocumentation = (sourceFile, context, cb) ->
  fs.readFile sourceFile, "utf-8", (err, code) ->
    throw err if err
    sections = parse get_language(sourceFile), code
    generate_source_html sourceFile, context, sections
    cb()

# Given a string of source code, parse out each comment and the code that
# follows it, and create an individual **section** for it.
parse = (language, data) ->
  lines = data.split '\n'
  sections = []
  docs = code = ''

  inMulti = no
  hasCode = no
  multiAccum = ''

  save = (docs, code) ->
    sections.push {
      docs_text: docs
      docs_html: showdown.makeHtml docs
      code_text: code
      code_html: highlight_start + code + highlight_end
    }

  for line in lines

    # Multi line comment
    if line.match(language.multi_start_matcher) or inMulti

      ## Start of a new section, save the old section
      if hasCode
        save docs, code
        docs = code = ''
        hasCode = no

      # Found the start of a multiline comment.
      # Begin accumulating lines until we reach the end of the comment block.
      inMulti = yes
      multiAccum += line + '\n'

      # If we reached the end of a multiline comment,
      # set inMulti to false and reset multiAccum
      if line.match(language.multi_end_matcher)
        inMulti = no
        docs = multiAccum
        multiAccum = ''

    # Single line comment
    else if line.match(language.comment_matcher) and not line.match(language.comment_filter)
      if hasCode
        save docs, code
        hasCode = docs = code = ''
      docs += line.replace(language.comment_matcher, '') + '\n'

    # Code
    else
      hasCode = yes
      code += line + '\n'

  save docs, code
  sections
   
# Once all of the code is finished highlighting, we can generate the HTML file
# and write out the documentation. Pass the completed sections into the template
# found in `resources/docco.jst`
generate_source_html = (source, context, sections) ->
  title = path.basename source
  dest  = destination source, context
  html  = docco_template {
    title: title, file_path: source, sections: sections, context: context, path: path, relative_base: relative_base
  }

  console.log "docco: #{source} -> #{dest}"
  write_file(dest, html)

generate_readme = (context, sources, package_json) ->
  title = "README"
  dest = "#{context.config.output_dir}/index.html"
  source = "README.md"

  # README.md template to be use to generate the main README file
  readme_template  = jade.compile fs.readFileSync(__dirname + '/../resources/readme.jade').toString(), { filename: __dirname + '/../resources/readme.jade' }
  readme_path = "#{process.cwd()}/#{source}"
  content_index_path = "#{process.cwd()}/#{context.config.content_dir}/content_index.md"
  
  # generate the content index if it exists under the content sources
  if file_exists(content_index_path)
    content_index = parse_markdown context, content_index_path
  else
    content_index = ""
 
  # parse the markdown the the readme 
  content = parse_markdown(context, readme_path) || "There is no #{source} for this project yet :( "
  
  # run cloc 
  cloc sources.join(" "), (code_stats) ->

    html = readme_template {
      title: title
      context: context
      content: content
      content_index: content_index
      file_path: source
      path: path
      relative_base: relative_base
      package_json: package_json
      code_stats: code_stats
      gravatar: gravatar
    }
    
    console.log "docco: #{source} -> #{dest}"
    write_file(dest, html)

generate_content = (context, dir) ->
  walker = walk.walk(dir, { followLinks: false })
  walker.on 'file', (root, fileStats, next) ->
    # only match files that end in *.md
    if fileStats.name.match(new RegExp ".md$")
      src = "#{root}/#{fileStats.name}"
      dest  = destination(src.replace(context.config.content_dir, ""), context)
      console.log "markdown: #{src} --> #{dest}"
      html = parse_markdown context, src
      html = content_template {
        title: fileStats.name, context: context, content: html, file_path: fileStats.name, path: path, relative_base: relative_base
      }
      write_file dest, html
    next()

# Write a file to the filesystem
write_file = (dest, contents) ->

    target_dir = path.dirname(dest)
    write_func = ->
      fs.writeFile dest, contents, (err) -> throw err if err

    fs.stat target_dir, (err, stats) ->
      throw err if err and err.code != 'ENOENT'

      return write_func() unless err

      if err
        exec "mkdir -p #{target_dir}", (err) ->
          throw err if err
          write_func()


# Parse a markdown file and return the HTML 
parse_markdown = (context, src) ->
  markdown = fs.readFileSync(src).toString()
  return showdown.makeHtml markdown

cloc = (paths, callback) ->
  exec "#{__dirname}/../vendor/cloc.pl --quiet --read-lang-def=#{__dirname}/../resources/cloc_definitions.txt #{paths}", (err, stdout) ->
    console.log "Calculating project stats failed #{err}" if err
    callback stdout

# A list of the languages that Docco supports, mapping the file extension to
# the name of the Pygments lexer and the symbol that indicates a comment. To
# add another language to Docco's repertoire, add it here.
languages =
  '.css':
    name: 'css', symbol: '//', multi_start: "/*", multi_end: "*/"
  '.scss':
    name: 'scss', symbol: '//', multi_start: "/*", multi_end: "*/"
  '.less':
    name: 'less', symbol: '//', multi_start: "/*", multi_end: "*/"

{'fraction_non_alphanumeric': 0.112394558857056, 'fraction_numerical': 0.00022798084960863287, 'mean_word_length': 3.0958605664488017, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5126288', 'n_tokens_mistral': 4027, 'n_tokens_neox': 3899, 'n_words': 1292}