<commit_before>require 'asciidoctor'

# fill in the db from a local git clone
task :local_index => :environment do
  dir     = ENV["GIT_REPO"]
  rebuild = ENV['REBUILD_DOC']
  rerun   = ENV['RERUN'] || false

  Dir.chdir(dir) do
    # find all tags
    tags = `git tag | egrep 'v1|v2'`.strip.split("\n")
    tags = tags.select { |tag| tag =~ /v\d([\.\d])+$/ }  # just get release tags

    if rebuild
      tags = tags.select { |t| t == rebuild }
      rerun = true
    end

    # for each tag, get a date and a list of file/shas
    tags.sort.each do |tag|

      puts tag

      stag = Version.where(:name => tag.gsub('v','')).first
      next if stag && !rerun

      stag = Version.where(:name => tag.gsub('v','')).first_or_create

      # extract metadata
      commit_sha = `git rev-parse #{tag}`.chomp
      tree_sha = `git rev-parse #{tag}^{tree}`.chomp
      tagger = `git cat-file commit #{tag} | grep committer`.chomp.split(' ')
      tz = tagger.pop
      ts = tagger.pop
      ts = Time.at(ts.to_i)

      # save metadata
      puts "#{tag}: #{ts}, #{commit_sha[0, 8]}, #{tree_sha[0, 8]}"
      stag.commit_sha = commit_sha
      stag.tree_sha = tree_sha
      stag.committed = ts
      stag.save

      # find all the doc entries
      entries = `git ls-tree #{tag}:Documentation`.strip.split("\n")
      tree = entries.map do |e|
        mode, type, sha, path = e.split(' ')
        [path, sha, type]
      end
      tree = tree.select { |t| t.first =~
          /^(
              git.* |
              everyday  |
              howto-index |
              user-manual |
              diff.* |
              fetch.* |
              merge.* |
              rev.* |
              pretty.* |
              pull.*
          )\.txt/x
      }

      puts "Found #{tree.size} entries"
      doc_limit = ENV['ONLY_BUILD_DOC']

      # generate command-list content
      categories = {}
      if system("git cat-file -e #{tag}:command-list.txt > /dev/null 2>&1")
        cmd_list = `git cat-file blob #{tag}:command-list.txt`.match(/(### command list.*|# command name.*)/m)[0].split("\n").reject{|l| l =~ /^#/}.inject({}) do |list, cmd|
          name, kind, attr = cmd.split(/\s+/)
          list[kind] ||= []
          list[kind] << [name, attr]
          list
        end

        categories = cmd_list.keys.inject({}) do |list, category|
          links = cmd_list[category].map do |cmd, attr|
            if match = `git cat-file blob #{tag}:Documentation/#{cmd}.txt`.match(/NAME\n----\n\S+ - (.*)$/)
              "linkgit:#{cmd}[1]::\n\t#{attr == 'deprecated' ? '(deprecated) ' : ''}#{match[1]}\n"
            end
          end
          list.merge!("cmds-#{category}.txt" => links.compact.join("\n"))
        end
      end

      def expand!(content, tag, categories)
        content.gsub!(/include::(\S+)\.txt/) do |line|
          line.gsub!("include::", "")
          new_content = categories[line] || `git cat-file blob #{tag}:Documentation/#{line}`
          expand!(new_content, tag, categories)
        end
        return content
      end

      tree.each do |entry|
        path, sha, type = entry
        path = path.gsub('.txt', '')
        next if doc_limit && path !~ /#{doc_limit}/
        file = DocFile.where(:name => path).first_or_create

        puts "   build: #{path}"

        content = `git cat-file blob #{sha}`.chomp
        expand!(content, tag, categories)
        content.gsub!(/link:technical\/(.*?)\.html\[(.*?)\]/, "link:\1\[\2\]")
<commit_msg>tasks/local_index: build technical pages<commit_after>require 'asciidoctor'

# fill in the db from a local git clone
task :local_index => :environment do
  dir     = ENV["GIT_REPO"]
  rebuild = ENV['REBUILD_DOC']
  rerun   = ENV['RERUN'] || false

  Dir.chdir(dir) do
    # find all tags
    tags = `git tag | egrep 'v1|v2'`.strip.split("\n")
    tags = tags.select { |tag| tag =~ /v\d([\.\d])+$/ }  # just get release tags

    if rebuild
      tags = tags.select { |t| t == rebuild }
      rerun = true
    end

    # for each tag, get a date and a list of file/shas
    tags.sort.each do |tag|

      puts tag

      stag = Version.where(:name => tag.gsub('v','')).first
      next if stag && !rerun

      stag = Version.where(:name => tag.gsub('v','')).first_or_create

      # extract metadata
      commit_sha = `git rev-parse #{tag}`.chomp
      tree_sha = `git rev-parse #{tag}^{tree}`.chomp
      tagger = `git cat-file commit #{tag} | grep committer`.chomp.split(' ')
      tz = tagger.pop
      ts = tagger.pop
      ts = Time.at(ts.to_i)

      # save metadata
      puts "#{tag}: #{ts}, #{commit_sha[0, 8]}, #{tree_sha[0, 8]}"
      stag.commit_sha = commit_sha
      stag.tree_sha = tree_sha
      stag.committed = ts
      stag.save

      # find all the doc entries
      entries = `git ls-tree -r #{tag}:Documentation`.strip.split("\n")
      tree = entries.map do |e|
        mode, type, sha, path = e.split(' ')
        [path, sha, type]
      end
      tree = tree.select { |t| t.first =~
          /^(
              git.* |
              everyday  |
              howto-index |
              user-manual |
              diff.* |
              fetch.* |
              merge.* |
              rev.* |
              pretty.* |
              pull.* |
              technical\/.*
          )\.txt/x
      }

      puts "Found #{tree.size} entries"
      doc_limit = ENV['ONLY_BUILD_DOC']

      # generate command-list content
      categories = {}
      if system("git cat-file -e #{tag}:command-list.txt > /dev/null 2>&1")
        cmd_list = `git cat-file blob #{tag}:command-list.txt`.match(/(### command list.*|# command name.*)/m)[0].split("\n").reject{|l| l =~ /^#/}.inject({}) do |list, cmd|
          name, kind, attr = cmd.split(/\s+/)
          list[kind] ||= []
          list[kind] << [name, attr]
          list
        end

        categories = cmd_list.keys.inject({}) do |list, category|
          links = cmd_list[category].map do |cmd, attr|
            if match = `git cat-file blob #{tag}:Documentation/#{cmd}.txt`.match(/NAME\n----\n\S+ - (.*)$/)
              "linkgit:#{cmd}[1]::\n\t#{attr == 'deprecated' ? '(deprecated) ' : ''}#{match[1]}\n"
            end
          end
          list.merge!("cmds-#{category}.txt" => links.compact.join("\n"))
        end
      end

      def expand!(content, tag, categories)
        content.gsub!(/include::(\S+)\.txt/) do |line|
          line.gsub!("include::", "")
          new_content = categories[line] || `git cat-file blob #{tag}:Documentation/#{line}`
          expand!(new_content, tag, categories)
        end
        return content
      end

      tree.each do |entry|
        path, sha, type = entry
        path = File.basename(path, '.txt' )
        next if doc_limit && path !~ /#{doc_limit}/
        file = DocFile.where(:name => path).first_or_create

        puts "   build: #{path}"

        content = `git cat-file blob #{sha}`.chomp
        expand!(content, tag, categories)
        content.gsub!(/link:technical\/(.*?)\.html\[(.*?)\]/, "link:\1\[\2\]")

{'fraction_non_alphanumeric': 0.17596140201504187, 'fraction_numerical': 0.0036895132680573295, 'mean_word_length': 1.892080426754206, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7430238', 'n_tokens_mistral': 2344, 'n_tokens_neox': 2313, 'n_words': 548}