<commit_before>require 'json'
require 'tempfile'
require 'version_sorter'
require 'rocco'
require 'docurium/version'
require 'docurium/layout'
require 'docurium/debug'
require 'libdetect'
require 'docurium/docparser'
require 'pp'
require 'rugged'
require 'redcarpet'
require 'redcarpet/compat'
require 'parallel'
require 'thread'

# Markdown expects the old redcarpet compat API, so let's tell it what
# to use
Rocco::Markdown = RedcarpetCompat

class Docurium
  attr_accessor :branch, :output_dir, :data, :head_data

  def initialize(config_file, cli_options = {}, repo = nil)
    raise "You need to specify a config file" if !config_file
    raise "You need to specify a valid config file" if !valid_config(config_file)
    @sigs = {}
    @head_data = nil
    @repo = repo || Rugged::Repository.discover(config_file)
    @cli_options = cli_options
  end

  def init_data(version = 'HEAD')
    data = {:files => [], :functions => {}, :callbacks => {}, :globals => {}, :types => {}, :prefix => ''}
    data[:prefix] = option_version(version, 'input', '')
    data
  end

  def option_version(version, option, default = nil)
    if @options['legacy']
      if valhash = @options['legacy'][option]
        valhash.each do |value, versions|
          return value if versions.include?(version)
        end
      end
    end
    opt = @options[option]
    opt = default if !opt
    opt
  end

  def format_examples!(data, version)
    examples = []
    if ex = option_version(version, 'examples')
      if subtree = find_subtree(version, ex) # check that it exists
        index = Rugged::Index.new
        index.read_tree(subtree)

        files = []
        index.each do |entry|
          next unless entry[:path].match(/\.c$/)
          files << entry[:path]
        end

        files.each do |file|
          # highlight, roccoize and link
          rocco = Rocco.new(file, files, {:language => 'c'}) do
            ientry = index[file]
            blob = @repo.lookup(ientry[:oid])
            blob.content
          end

          extlen = -(File.extname(file).length + 1)
          rf_path = file[0..extlen] + '.html'
          rel_path = "ex/#{version}/#{rf_path}"

          rocco_layout = Rocco::Layout.new(rocco, @tf)
          # find out how deep our file is so we can use the right
          # number of ../ in the path
          depth = rel_path.count('/') - 1
          if depth == 0
            rocco_layout[:dirsup] = "./"
          else
            rocco_layout[:dirsup] = "../"*depth
          end

          rocco_layout.version = version
          rf = rocco_layout.render


          # look for function names in the examples and link
          id_num = 0
          data[:functions].each do |f, fdata|
            rf.gsub!(/#{f}([^\w])/) do |fmatch|
              extra = $1
              id_num += 1
              name = f + '-' + id_num.to_s
              # save data for cross-link
              data[:functions][f][:examples] ||= {}
              data[:functions][f][:examples][file] ||= []
              data[:functions][f][:examples][file] << rel_path + '#' + name
              "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
            end
          end

          # write example to the repo
          sha = @repo.write(rf, :blob)
          examples << [rel_path, sha]

          data[:examples] ||= []
          data[:examples] << [file, rel_path]
        end
      end
    end

    examples
  end

  def generate_doc_for(version)
    index = Rugged::Index.new
    read_subtree(index, version, option_version(version, 'input', ''))

    data = parse_headers(index, version)
    examples = format_examples!(data, version)
    [data, examples]
  end

  def process_project(versions)
    nversions = versions.count
    Parallel.each_with_index(versions, finish: -> (version, index, result) do
      data, examples = result
      # There's still some work we need to do serially
      tally_sigs!(version, data)
      force_utf8(data)

      puts "Adding documentation for #{version} [#{index}/#{nversions}]"

      # Store it so we can show it at the end
      @head_data = data if version == 'HEAD'

      yield index, version, result if block_given?

    end) do |version, index|
      puts "Generating documentation for #{version} [#{index}/#{nversions}]"
      generate_doc_for(version)
    end
  end

  def generate_docs
    output_index = Rugged::Index.new
    write_site(output_index)
    @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
    versions = get_versions
    versions << 'HEAD'
    # If the user specified versions, validate them and overwrite
    if !(vers = (@cli_options[:for] || [])).empty?
      vers.each do |v|
        next if versions.include?(v)
        puts "Unknown version #{v}"
        exit(false)
      end
      versions = vers
    end

    process_project(versions) do |i, version, result|
      data, examples = result
      sha = @repo.write(data.to_json, :blob)<commit_msg>Add clean exit if no git user configuration is set<commit_after>require 'json'
require 'tempfile'
require 'version_sorter'
require 'rocco'
require 'docurium/version'
require 'docurium/layout'
require 'docurium/debug'
require 'libdetect'
require 'docurium/docparser'
require 'pp'
require 'rugged'
require 'redcarpet'
require 'redcarpet/compat'
require 'parallel'
require 'thread'

# Markdown expects the old redcarpet compat API, so let's tell it what
# to use
Rocco::Markdown = RedcarpetCompat

class Docurium
  attr_accessor :branch, :output_dir, :data, :head_data

  def initialize(config_file, cli_options = {}, repo = nil)
    raise "You need to specify a config file" if !config_file
    raise "You need to specify a valid config file" if !valid_config(config_file)
    @sigs = {}
    @head_data = nil
    @repo = repo || Rugged::Repository.discover(config_file)
    @cli_options = cli_options
  end

  def init_data(version = 'HEAD')
    data = {:files => [], :functions => {}, :callbacks => {}, :globals => {}, :types => {}, :prefix => ''}
    data[:prefix] = option_version(version, 'input', '')
    data
  end

  def option_version(version, option, default = nil)
    if @options['legacy']
      if valhash = @options['legacy'][option]
        valhash.each do |value, versions|
          return value if versions.include?(version)
        end
      end
    end
    opt = @options[option]
    opt = default if !opt
    opt
  end

  def format_examples!(data, version)
    examples = []
    if ex = option_version(version, 'examples')
      if subtree = find_subtree(version, ex) # check that it exists
        index = Rugged::Index.new
        index.read_tree(subtree)

        files = []
        index.each do |entry|
          next unless entry[:path].match(/\.c$/)
          files << entry[:path]
        end

        files.each do |file|
          # highlight, roccoize and link
          rocco = Rocco.new(file, files, {:language => 'c'}) do
            ientry = index[file]
            blob = @repo.lookup(ientry[:oid])
            blob.content
          end

          extlen = -(File.extname(file).length + 1)
          rf_path = file[0..extlen] + '.html'
          rel_path = "ex/#{version}/#{rf_path}"

          rocco_layout = Rocco::Layout.new(rocco, @tf)
          # find out how deep our file is so we can use the right
          # number of ../ in the path
          depth = rel_path.count('/') - 1
          if depth == 0
            rocco_layout[:dirsup] = "./"
          else
            rocco_layout[:dirsup] = "../"*depth
          end

          rocco_layout.version = version
          rf = rocco_layout.render


          # look for function names in the examples and link
          id_num = 0
          data[:functions].each do |f, fdata|
            rf.gsub!(/#{f}([^\w])/) do |fmatch|
              extra = $1
              id_num += 1
              name = f + '-' + id_num.to_s
              # save data for cross-link
              data[:functions][f][:examples] ||= {}
              data[:functions][f][:examples][file] ||= []
              data[:functions][f][:examples][file] << rel_path + '#' + name
              "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
            end
          end

          # write example to the repo
          sha = @repo.write(rf, :blob)
          examples << [rel_path, sha]

          data[:examples] ||= []
          data[:examples] << [file, rel_path]
        end
      end
    end

    examples
  end

  def generate_doc_for(version)
    index = Rugged::Index.new
    read_subtree(index, version, option_version(version, 'input', ''))

    data = parse_headers(index, version)
    examples = format_examples!(data, version)
    [data, examples]
  end

  def process_project(versions)
    nversions = versions.count
    Parallel.each_with_index(versions, finish: -> (version, index, result) do
      data, examples = result
      # There's still some work we need to do serially
      tally_sigs!(version, data)
      force_utf8(data)

      puts "Adding documentation for #{version} [#{index}/#{nversions}]"

      # Store it so we can show it at the end
      @head_data = data if version == 'HEAD'

      yield index, version, result if block_given?

    end) do |version, index|
      puts "Generating documentation for #{version} [#{index}/#{nversions}]"
      generate_doc_for(version)
    end
  end

  def generate_docs
    output_index = Rugged::Index.new
    write_site(output_index)
    @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
    versions = get_versions
    versions << 'HEAD'
    # If the user specified versions, validate them and overwrite
    if !(vers = (@cli_options[:for] || [])).empty?
      vers.each do |v|
        next if versions.include?(v)
        puts "Unknown version #{v}"
        exit(false)
      end
      versions = vers
    end

    if (@repo.config['user.name'].nil? || @repo.config['user.email'].nil?)
      puts "ERROR: 'user.name' or 'user.email' is not configured. Docurium will not be able to commit the documentation"
      exit(false)
    end

    process_project(versions) do |i, version, result|
      data, examples = result
      sha = @repo.write(data.to_json, :blob)
{'fraction_non_alphanumeric': 0.1318893830980468, 'fraction_numerical': 0.0015470895378070005, 'mean_word_length': 2.4214356599404563, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58053', 'n_tokens_mistral': 3231, 'n_tokens_neox': 3135, 'n_words': 866}