<commit_before>require "find"
require "fileutils"
require "logger"
require "pathname"

require_relative "buildindex"
require_relative "docconverter"
require_relative "docid"
require_relative "indexheadings"
require_relative "docinfo"
require_relative "buildgraph"

module Giblish

  # Parse a directory tree and convert all asciidoc files matching the
  # supplied critera to the supplied format
  class FileTreeConverter

    attr_reader :converter

    # Required options:
    #  srcDirRoot
    #  dstDirRoot
    #  resourceDir
    def initialize(options)
      @options = options.dup

      @paths = Giblish::PathManager.new(
          @options[:srcDirRoot],
          @options[:dstDirRoot],
          @options[:resourceDir],
          @options[:webRoot]
      )
      @processed_docs = []
      @converter = converter_factory
      @search_assets_path = @paths.dst_root_abs.realpath.join("search_assets")
    end


    # convert all adoc files
    # return true if all conversions went ok, false if at least one
    # failed
    def convert
      # collect all doc ids and enable replacement of known doc ids with
      # valid references to adoc files
      manage_doc_ids if @options[:resolveDocid]

      # register add-on for handling searchability
      manage_searchability if @options[:make_searchable]

      # traverse the src file tree and convert all files deemed as
      # adoc files
      conv_error = false
      Find.find(@paths.src_root_abs) do |path|
        p = Pathname.new(path)
        begin
          to_asciidoc(p) if adocfile? p
        rescue Exception => e
          str = "Error when converting file #{path.to_s}: #{e.message}\nBacktrace:\n"
          e.backtrace.each {|l| str << "   #{l}\n"}
          Giblog.logger.error {str}
          conv_error = true
        end
      end if @paths.src_root_abs.directory?

      # create necessary search assets if needed
      create_search_assets if @options[:make_searchable]

      # build index and other fancy stuff if not suppressed
      unless @options[:suppressBuildRef]
        # build a dependency graph (only if we resolve docids...)
        dep_graph_exist = if @options[:resolveDocid]
          if Giblish::GraphBuilderGraphviz.supported
            # gb = Giblish::GraphBuilderGraphviz.new @processed_docs, @paths, {extension: @converter.converter_options[:fileext]}
            gb = Giblish::GraphBuilderGraphviz.new @processed_docs, @paths, @converter.converter_options
            puts gb.source(@options[:make_searchable])
            errors = @converter.convert_str(
                gb.source(
                    @options[:make_searchable]
                ),
                @paths.dst_root_abs,
                "graph"
            )
            !errors
          else
            Giblog.logger.warn { "Lacking access to needed tools for generating a visual dependency graph." }
            Giblog.logger.warn { "The dependency graph will not be generated !!" }
            false
          end
        else
          false
        end

        # build a reference index
        adoc_logger = Giblish::AsciidoctorLogger.new Logger::Severity::WARN
        ib = index_factory
        @converter.convert_str(
            ib.source(
                dep_graph_exist,@options[:make_searchable]
            ),
            @paths.dst_root_abs,
            "index",
            logger: adoc_logger)

        # clean up cached files and adoc resources
        remove_diagram_temps if dep_graph_exist
        GC.start
      end
      conv_error
    end

    protected

    # get the correct index builder type depending on supplied
    # user options
    def index_factory
      raise "Internal logic error!" if @options[:suppressBuildRef]
      SimpleIndexBuilder.new(@processed_docs, @converter, @paths,
                             @options[:resolveDocid])
    end

    # get the correct converter type
    def converter_factory
      case @options[:format]
        when "html" then
          HtmlConverter.new @paths, @options
        when "pdf" then
          PdfConverter.new @paths, @options
        else
          raise ArgumentError, "Unknown conversion format: #{@options[:format]}"
      end
    end

    # creates a DocInfo instance, fills it with basic info and<commit_msg>refactor index and graph page to separate methods<commit_after>require "find"
require "fileutils"
require "logger"
require "pathname"

require_relative "buildindex"
require_relative "docconverter"
require_relative "docid"
require_relative "indexheadings"
require_relative "docinfo"
require_relative "buildgraph"

module Giblish

  # Parse a directory tree and convert all asciidoc files matching the
  # supplied critera to the supplied format
  class FileTreeConverter

    attr_reader :converter

    # Required options:
    #  srcDirRoot
    #  dstDirRoot
    #  resourceDir
    def initialize(options)
      @options = options.dup

      @paths = Giblish::PathManager.new(
          @options[:srcDirRoot],
          @options[:dstDirRoot],
          @options[:resourceDir],
          @options[:webRoot]
      )
      @processed_docs = []
      @converter = converter_factory
      @search_assets_path = @paths.dst_root_abs.realpath.join("search_assets")
    end


    # convert all adoc files
    # return true if all conversions went ok, false if at least one
    # failed
    def convert
      # collect all doc ids and enable replacement of known doc ids with
      # valid references to adoc files
      manage_doc_ids if @options[:resolveDocid]

      # register add-on for handling searchability
      manage_searchability if @options[:make_searchable]

      # traverse the src file tree and convert all files deemed as
      # adoc files
      conv_error = false
      Find.find(@paths.src_root_abs) do |path|
        p = Pathname.new(path)
        begin
          to_asciidoc(p) if adocfile? p
        rescue Exception => e
          str = "Error when converting file #{path.to_s}: #{e.message}\nBacktrace:\n"
          e.backtrace.each {|l| str << "   #{l}\n"}
          Giblog.logger.error {str}
          conv_error = true
        end
      end if @paths.src_root_abs.directory?

      # create necessary search assets if needed
      create_search_assets if @options[:make_searchable]

      # build index and other fancy stuff if not suppressed
      unless @options[:suppressBuildRef]
        # build a dependency graph (only if we resolve docids...)
        dep_graph_exist = @options[:resolveDocid] && build_graph_page

        # build a reference index
        build_index_page(dep_graph_exist)
      end
      conv_error
    end

    protected

    def build_graph_page
      if Giblish::GraphBuilderGraphviz.supported
        # gb = Giblish::GraphBuilderGraphviz.new @processed_docs, @paths, {extension: @converter.converter_options[:fileext]}
        gb = Giblish::GraphBuilderGraphviz.new @processed_docs, @paths, @converter.converter_options
        errors = @converter.convert_str(
            gb.source(
                @options[:make_searchable]
            ),
            @paths.dst_root_abs,
            "graph"
        )
        remove_diagram_temps unless errors
        !errors
      else
        Giblog.logger.warn { "Lacking access to needed tools for generating a visual dependency graph." }
        Giblog.logger.warn { "The dependency graph will not be generated !!" }
        false
      end
    end

    def build_index_page(dep_graph_exist)
      # build a reference index
      adoc_logger = Giblish::AsciidoctorLogger.new Logger::Severity::WARN
      ib = index_factory
      @converter.convert_str(
          ib.source(
              dep_graph_exist,@options[:make_searchable]
          ),
          @paths.dst_root_abs,
          "index",
          logger: adoc_logger
      )

      # clean up cached files and adoc resources
      GC.start
    end

    # get the correct index builder type depending on supplied
    # user options
    def index_factory
      raise "Internal logic error!" if @options[:suppressBuildRef]
      SimpleIndexBuilder.new(@processed_docs, @converter, @paths,
                             @options[:resolveDocid])
    end

    # get the correct converter type
    def converter_factory
      case @options[:format]
        when "html" then
          HtmlConverter.new @paths, @options
        when "pdf" then
          PdfConverter.new @paths, @options
        else
          raise ArgumentError, "Unknown conversion format: #{@options[:format]}"
      end
    end

    # creates a DocInfo instance, fills it with basic info and
{'fraction_non_alphanumeric': 0.07629491367242183, 'fraction_numerical': 0.0, 'mean_word_length': 2.3912183544303796, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1734268', 'n_tokens_mistral': 2379, 'n_tokens_neox': 2310, 'n_words': 678}