<commit_before>require 'json'
require 'tempfile'
require 'version_sorter'
require 'rocco'
require 'docurium/layout'
require 'docurium/cparser'
require 'pp'
require 'rugged'

class Docurium
  Version = VERSION = '0.0.5'

  attr_accessor :branch, :output_dir, :data

  def initialize(config_file)
    raise "You need to specify a config file" if !config_file
    raise "You need to specify a valid config file" if !valid_config(config_file)
    @sigs = {}
    @groups = {}
    repo_path = Rugged::Repository.discover('.')
    @repo = Rugged::Repository.new(repo_path)
    clear_data
  end

  def clear_data(version = 'HEAD')
    @data = {:files => [], :functions => {}, :globals => {}, :types => {}, :prefix => ''}
    @data[:prefix] = option_version(version, 'input', '')
  end

  def option_version(version, option, default = nil)
    if @options['legacy']
      if valhash = @options['legacy'][option]
        valhash.each do |value, versions|
          return value if versions.include?(version)
        end
      end
    end
    opt = @options[option]
    opt = default if !opt
    opt
  end

  def generate_docs
    out "* generating docs"
    outdir = mkdir_temp
    copy_site(outdir)
    versions = get_versions
    versions << 'HEAD'
    versions.each do |version|
      out "  - processing version #{version}"
      workdir = mkdir_temp
      Dir.chdir(workdir) do
        clear_data(version)
        checkout(version, workdir)
        parse_headers
        tally_sigs(version)
      end

      tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
      if ex = option_version(version, 'examples')
        workdir = mkdir_temp
        Dir.chdir(workdir) do
          with_git_env(workdir) do
            `git rev-parse #{version}:#{ex} 2>&1` # check that it exists
            if $?.exitstatus == 0
              out "  - processing examples for #{version}"
              `git read-tree #{version}:#{ex}`
              `git checkout-index -a`

              files = []
              Dir.glob("**/*.c") do |file|
                next if !File.file?(file)
                files << file
              end
              files.each do |file|
                out "    # #{file}"

                # highlight, roccoize and link
                rocco = Rocco.new(file, files, {:language => 'c'})
                rocco_layout = Rocco::Layout.new(rocco, tf)
                rocco_layout.version = version
                rf = rocco_layout.render

                rf_path = File.basename(file).split('.')[0..-2].join('.') + '.html'
                rel_path = "ex/#{version}/#{rf_path}"
                rf_path = File.join(outdir, rel_path)

                # look for function names in the examples and link
                id_num = 0
                @data[:functions].each do |f, fdata|
                  rf.gsub!(/#{f}([^\w])/) do |fmatch|
                    extra = $1
                    id_num += 1
                    name = f + '-' + id_num.to_s
                    # save data for cross-link
                    @data[:functions][f][:examples] ||= {}
                    @data[:functions][f][:examples][file] ||= []
                    @data[:functions][f][:examples][file] << rel_path + '#' + name
                    "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
                  end
                end

                # write example to docs directory
                FileUtils.mkdir_p(File.dirname(rf_path))
                File.open(rf_path, 'w+') do |f|
                  @data[:examples] ||= []
                  @data[:examples] << [file, rel_path]
                  f.write(rf)
                end
              end
            end
          end
        end

        if version == 'HEAD'
          show_warnings
        end
      end

      File.open(File.join(outdir, "#{version}.json"), 'w+') do |f|
        f.write(@data.to_json)
      end
    end

    Dir.chdir(outdir) do
      project = {
        :versions => versions.reverse,
        :github   => @options['github'],
        :name     => @options['name'],
        :signatures => @sigs,
        :groups   => @groups
      }
      File.open("project.json", 'w+') do |f|
        f.write(project.to_json)
      end
    end

    if br = @options['branch']
      out "* writing to branch #{br}"
      ref = "refs/heads/#{br}"
      with_git_env(outdir) do
        psha = `git rev-parse #{ref}`.chomp
        `git add -A`
        tsha = `git write-tree`.chomp
        puts "\twrote tree   #{tsha}"
        if(psha == ref)
          csha = `echo 'generated docs' | git commit-tree #{tsha}`.chomp
        else
          csha = `echo 'generated docs' | git commit-tree #{tsha} -p #{psha}`.chomp
        end
        puts "\twrote commit #{csha}"
        `git update-ref -m 'generated docs' #{ref} #{csha}`
        puts "\tupdated #{br}"
      end
    else
      final_dir = File.join(@project_dir, @options['output'] || 'docs')
      out "* output html in #{final_dir}"
      FileUtils.mkdir_p(final_dir)
      Dir.chdir(final_dir) do
        FileUtils.cp_r(File.join(outdir, '.'), '.') <commit_msg>Update the docs branch via rugged<commit_after>require 'json'
require 'tempfile'
require 'version_sorter'
require 'rocco'
require 'docurium/layout'
require 'docurium/cparser'
require 'pp'
require 'rugged'

class Docurium
  Version = VERSION = '0.0.5'

  attr_accessor :branch, :output_dir, :data

  def initialize(config_file)
    raise "You need to specify a config file" if !config_file
    raise "You need to specify a valid config file" if !valid_config(config_file)
    @sigs = {}
    @groups = {}
    repo_path = Rugged::Repository.discover('.')
    @repo = Rugged::Repository.new(repo_path)
    clear_data
  end

  def clear_data(version = 'HEAD')
    @data = {:files => [], :functions => {}, :globals => {}, :types => {}, :prefix => ''}
    @data[:prefix] = option_version(version, 'input', '')
  end

  def option_version(version, option, default = nil)
    if @options['legacy']
      if valhash = @options['legacy'][option]
        valhash.each do |value, versions|
          return value if versions.include?(version)
        end
      end
    end
    opt = @options[option]
    opt = default if !opt
    opt
  end

  def generate_docs
    out "* generating docs"
    outdir = mkdir_temp
    copy_site(outdir)
    versions = get_versions
    versions << 'HEAD'
    versions.each do |version|
      out "  - processing version #{version}"
      workdir = mkdir_temp
      Dir.chdir(workdir) do
        clear_data(version)
        checkout(version, workdir)
        parse_headers
        tally_sigs(version)
      end

      tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
      if ex = option_version(version, 'examples')
        workdir = mkdir_temp
        Dir.chdir(workdir) do
          with_git_env(workdir) do
            `git rev-parse #{version}:#{ex} 2>&1` # check that it exists
            if $?.exitstatus == 0
              out "  - processing examples for #{version}"
              `git read-tree #{version}:#{ex}`
              `git checkout-index -a`

              files = []
              Dir.glob("**/*.c") do |file|
                next if !File.file?(file)
                files << file
              end
              files.each do |file|
                out "    # #{file}"

                # highlight, roccoize and link
                rocco = Rocco.new(file, files, {:language => 'c'})
                rocco_layout = Rocco::Layout.new(rocco, tf)
                rocco_layout.version = version
                rf = rocco_layout.render

                rf_path = File.basename(file).split('.')[0..-2].join('.') + '.html'
                rel_path = "ex/#{version}/#{rf_path}"
                rf_path = File.join(outdir, rel_path)

                # look for function names in the examples and link
                id_num = 0
                @data[:functions].each do |f, fdata|
                  rf.gsub!(/#{f}([^\w])/) do |fmatch|
                    extra = $1
                    id_num += 1
                    name = f + '-' + id_num.to_s
                    # save data for cross-link
                    @data[:functions][f][:examples] ||= {}
                    @data[:functions][f][:examples][file] ||= []
                    @data[:functions][f][:examples][file] << rel_path + '#' + name
                    "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
                  end
                end

                # write example to docs directory
                FileUtils.mkdir_p(File.dirname(rf_path))
                File.open(rf_path, 'w+') do |f|
                  @data[:examples] ||= []
                  @data[:examples] << [file, rel_path]
                  f.write(rf)
                end
              end
            end
          end
        end

        if version == 'HEAD'
          show_warnings
        end
      end

      File.open(File.join(outdir, "#{version}.json"), 'w+') do |f|
        f.write(@data.to_json)
      end
    end

    Dir.chdir(outdir) do
      project = {
        :versions => versions.reverse,
        :github   => @options['github'],
        :name     => @options['name'],
        :signatures => @sigs,
        :groups   => @groups
      }
      File.open("project.json", 'w+') do |f|
        f.write(project.to_json)
      end
    end

    if br = @options['branch']
      out "* writing to branch #{br}"
      refname = "refs/heads/#{br}"
      with_git_env(outdir) do
        `git add -A`
        tsha = `git write-tree`.chomp
        puts "\twrote tree   #{tsha}"
        ref = Rugged::Reference.lookup(@repo, refname)
        user = { :name => @repo.config['user.name'], :email => @repo.config['user.email'], :time => Time.now }
        options = {}
        options[:tree] = tsha
        options[:author] = user
        options[:committer] = user
        options[:message] = 'generated docs'
        options[:parents] = ref ? [ref.target] : []
        options[:update_ref] = refname
        csha = Rugged::Commit.create(@repo, options)
        puts "\twrote commit #{csha}"
        puts "\tupdated #{br}"
      end
    else
      final_dir = File.join(@project_dir, @options['output'] || 'docs')
      out "* output html in #{final_dir}"
      FileUtils.mkdir_p(final_dir)
      Dir.chdir(final_dir) do
        FileUtils.cp_r(File.join(outdir, '.'), '.') 
{'fraction_non_alphanumeric': 0.1444602948169282, 'fraction_numerical': 0.0020922491678554444, 'mean_word_length': 1.8020250466293632, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2681297', 'n_tokens_mistral': 3302, 'n_tokens_neox': 3185, 'n_words': 744}