<commit_before>require 'json'
require 'tempfile'
require 'version_sorter'
require 'rocco'
require 'docurium/version'
require 'docurium/layout'
require 'libdetect'
require 'docurium/docparser'
require 'pp'
require 'rugged'
require 'redcarpet'
require 'thread'

# Markdown expects the old redcarpet compat API, so let's tell it what
# to use
Rocco::Markdown = RedcarpetCompat

class Docurium
  attr_accessor :branch, :output_dir, :data

  def initialize(config_file, repo = nil)
    raise "You need to specify a config file" if !config_file
    raise "You need to specify a valid config file" if !valid_config(config_file)
    @sigs = {}
    if repo
      @repo = repo
    else
      repo_path = Rugged::Repository.discover('.')
      @repo = Rugged::Repository.new(repo_path)
    end
  end

  def init_data(version = 'HEAD')
    data = {:files => [], :functions => {}, :globals => {}, :types => {}, :prefix => ''}
    data[:prefix] = option_version(version, 'input', '')
    data
  end

  def option_version(version, option, default = nil)
    if @options['legacy']
      if valhash = @options['legacy'][option]
        valhash.each do |value, versions|
          return value if versions.include?(version)
        end
      end
    end
    opt = @options[option]
    opt = default if !opt
    opt
  end

  def format_examples!(data, version)
    examples = []
    if ex = option_version(version, 'examples')
      if subtree = find_subtree(version, ex) # check that it exists
        index = Rugged::Index.new
        index.read_tree(subtree)

        files = []
        index.each do |entry|
          next unless entry[:path].match(/\.c$/)
          files << entry[:path]
        end

        files.each do |file|
          # highlight, roccoize and link
          rocco = Rocco.new(file, files, {:language => 'c'}) do
            ientry = index[file]
            blob = @repo.lookup(ientry[:oid])
            blob.content
          end
          rocco_layout = Rocco::Layout.new(rocco, @tf)
          rocco_layout.version = version
          rf = rocco_layout.render

          extlen = -(File.extname(file).length + 1)
          rf_path = file[0..extlen] + '.html'
          rel_path = "ex/#{version}/#{rf_path}"

          # look for function names in the examples and link
          id_num = 0
          data[:functions].each do |f, fdata|
            rf.gsub!(/#{f}([^\w])/) do |fmatch|
              extra = $1
              id_num += 1
              name = f + '-' + id_num.to_s
              # save data for cross-link
              data[:functions][f][:examples] ||= {}
              data[:functions][f][:examples][file] ||= []
              data[:functions][f][:examples][file] << rel_path + '#' + name
              "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
            end
          end

          # write example to the repo
          sha = @repo.write(rf, :blob)
          examples << [rel_path, sha]

          data[:examples] ||= []
          data[:examples] << [file, rel_path]
        end
      end
    end

    examples
  end

  def generate_doc_for(version)
    index = Rugged::Index.new
    read_subtree(index, version, option_version(version, 'input', ''))
    data = parse_headers(index, version)
    data
  end

  def generate_docs
    output_index = Rugged::Index.new
    write_site(output_index)
    @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
    versions = get_versions
    versions << 'HEAD'
    nversions = versions.size
    output = Queue.new
    worker = Thread.new do
      versions.each do |version|

        data = generate_doc_for(version)
        examples = format_examples!(data, version)

        output << [version, data, examples]
      end
    end

    print "Generating documentation [0/#{nversions}]\r"
    for i in 1..nversions
      version, data, examples = output.pop

      # There's still some work we need to do serially
      tally_sigs!(version, data)
      sha = @repo.write(data.to_json, :blob)

      print "Generating documentation [#{i}/#{nversions}]\r"

      if version == 'HEAD'
        puts ''
        show_warnings(data)
      end

      output_index.add(:path => "#{version}.json", :oid => sha, :mode => 0100644)
      examples.each do |path, id|
        output_index.add(:path => path, :oid => id, :mode => 0100644)
      end
    end

    project = {
      :versions => versions.reverse,
      :github   => @options['github'],
      :name     => @options['name'],
      :signatures => @sigs,
    }
    sha = @repo.write(project.to_json, :blob)
    output_index.add(:path => "project.json", :oid => sha, :mode => 0100644)

    br = @options['branch']
    out "* writing to branch #{br}"
    refname = "refs/heads/#{br}"
    tsha = output_index.write_tree(@repo)
    puts "\twrote tree   #{tsha}"
    ref = Rugged::Reference.lookup(@repo, refname)
    user = { :name => @repo.config['user.name'], :email => @repo.config['user.email'], :time => Time.now }
    options = {}
    options[:tree] = tsha<commit_msg>Parallelise doc generation<commit_after>require 'json'
require 'tempfile'
require 'version_sorter'
require 'rocco'
require 'docurium/version'
require 'docurium/layout'
require 'libdetect'
require 'docurium/docparser'
require 'pp'
require 'rugged'
require 'redcarpet'
require 'thread'

# Markdown expects the old redcarpet compat API, so let's tell it what
# to use
Rocco::Markdown = RedcarpetCompat

class Docurium
  attr_accessor :branch, :output_dir, :data

  def initialize(config_file, repo = nil)
    raise "You need to specify a config file" if !config_file
    raise "You need to specify a valid config file" if !valid_config(config_file)
    @sigs = {}
    if repo
      @repo = repo
    else
      repo_path = Rugged::Repository.discover('.')
      @repo = Rugged::Repository.new(repo_path)
    end
  end

  def init_data(version = 'HEAD')
    data = {:files => [], :functions => {}, :globals => {}, :types => {}, :prefix => ''}
    data[:prefix] = option_version(version, 'input', '')
    data
  end

  def option_version(version, option, default = nil)
    if @options['legacy']
      if valhash = @options['legacy'][option]
        valhash.each do |value, versions|
          return value if versions.include?(version)
        end
      end
    end
    opt = @options[option]
    opt = default if !opt
    opt
  end

  def format_examples!(data, version)
    examples = []
    if ex = option_version(version, 'examples')
      if subtree = find_subtree(version, ex) # check that it exists
        index = Rugged::Index.new
        index.read_tree(subtree)

        files = []
        index.each do |entry|
          next unless entry[:path].match(/\.c$/)
          files << entry[:path]
        end

        files.each do |file|
          # highlight, roccoize and link
          rocco = Rocco.new(file, files, {:language => 'c'}) do
            ientry = index[file]
            blob = @repo.lookup(ientry[:oid])
            blob.content
          end
          rocco_layout = Rocco::Layout.new(rocco, @tf)
          rocco_layout.version = version
          rf = rocco_layout.render

          extlen = -(File.extname(file).length + 1)
          rf_path = file[0..extlen] + '.html'
          rel_path = "ex/#{version}/#{rf_path}"

          # look for function names in the examples and link
          id_num = 0
          data[:functions].each do |f, fdata|
            rf.gsub!(/#{f}([^\w])/) do |fmatch|
              extra = $1
              id_num += 1
              name = f + '-' + id_num.to_s
              # save data for cross-link
              data[:functions][f][:examples] ||= {}
              data[:functions][f][:examples][file] ||= []
              data[:functions][f][:examples][file] << rel_path + '#' + name
              "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
            end
          end

          # write example to the repo
          sha = @repo.write(rf, :blob)
          examples << [rel_path, sha]

          data[:examples] ||= []
          data[:examples] << [file, rel_path]
        end
      end
    end

    examples
  end

  def generate_doc_for(version)
    index = Rugged::Index.new
    read_subtree(index, version, option_version(version, 'input', ''))
    data = parse_headers(index, version)
    data
  end

  def generate_docs
    output_index = Rugged::Index.new
    write_site(output_index)
    @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
    versions = get_versions
    versions << 'HEAD'
    nversions = versions.size
    output = Queue.new
    pipes = {}
    versions.each do |version|
      # We don't need to worry about joining since this process is
      # going to die immediately
      read, write = IO.pipe
      pid = Process.fork do
        read.close

        data = generate_doc_for(version)
        examples = format_examples!(data, version)

        Marshal.dump([version, data, examples], write)
        write.close
      end

      pipes[pid] = read
      write.close
    end

    print "Generating documentation [0/#{nversions}]\r"
    head_data = nil

    # This may seem odd, but we need to keep reading from the pipe or
    # the buffer will fill and they'll block and never exit. Therefore
    # we can't rely on Process.wait to tell us when the work is
    # done. Instead read from all the pipes concurrently and send the
    # ruby objects through the queue.
    Thread.abort_on_exception = true
    pipes.each do |pid, read|
      Thread.new do
        result = read.read
        output << Marshal.load(result)
      end
    end

    for i in 1..nversions
      version, data, examples = output.pop

      # There's still some work we need to do serially
      tally_sigs!(version, data)
      sha = @repo.write(data.to_json, :blob)

      print "Generating documentation [#{i}/#{nversions}]\r"

      # Store it so we can show it at the end
      if version == 'HEAD'
        head_data = data
      end

      output_index.add(:path => "#{version}.json", :oid => sha, :mode => 0100644)
      examples.each do |path, id|
        output_index.add(:path => path, :oid => id, :mode => 0100644)
      end

      if head_data
        puts ''
        show_warnings(data)
      end

    end

    project = {
      :versions => versions.reverse,
      :github   => @options['github'],
      :name     => @options['name'],
      :signatures => @sigs,
    }
    sha = @repo.write(project.to_json, :blob)
    output_index.add(:path => "project.json", :oid => sha, :mode => 0100644)

    br = @options['branch']
    out "* writing to branch #{br}"
    refname = "refs/heads/#{br}"
    tsha = output_index.write_tree(@repo)
    puts "\twrote tree   #{tsha}"
    ref = Rugged::Reference.lookup(@repo, refname)
    user = { :name => @repo.config['user.name'], :email => @repo.config['user.email'], :time => Time.now }
    options = {}
    options[:tree] = tsha
{'fraction_non_alphanumeric': 0.13705583756345177, 'fraction_numerical': 0.005076142131979695, 'mean_word_length': 2.3698839340256566, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4069167', 'n_tokens_mistral': 3521, 'n_tokens_neox': 3397, 'n_words': 901}