<commit_before>#!/usr/bin/ruby

require 'fileutils'
require 'net/ftp'
require 'open3'
require 'pp'
require 'rubygems'
require 'rake'
# require 'progress'

class String
  def /(s)
    File.join(self, s)
  end
end

BASE_PATH = File.expand_path(File.dirname(__FILE__))
DOCS_PATH = BASE_PATH / 'docs'
SOURSES_PATH = BASE_PATH / 'sources'

class RdocAll
  class << self
    def update_sources(options = {})
      Base.update_all_sources(options)
    end

    def build_documentation(options = {})
      Base.build_documentation(options)
    end
  end
end

class RdocAll::Base
  class << self
    def inherited(subclass)
      (@subclasses ||= []) << subclass
    end

    def update_all_sources(options = {})
      Dir.chdir(SOURSES_PATH) do
        @subclasses.each do |subclass|
          subclass.update_sources(options)
        end
      end
    end

    def build_documentation(options = {})
      @rdoc_tasks = []
      Dir.chdir(SOURSES_PATH) do
        @subclasses.each do |subclass|
          subclass.add_rdoc_tasks
        end
      end
    end

  protected

    def add_rdoc_task(base_path, source_pathes = [])
      p [base_path, source_pathes.length]
      # Dir.chdir(path) do
      #   puts "Building #{path} documentation"
      #   Dir.rmdir(DOC_DIR / path) if File.directory?(DOC_DIR / path) && Dir[DOC_DIR / path / '*'].empty?
      #   system('hanna', '-o', DOC_DIR / path, '-t', path.sub('s/', ' — '), *pathes)
      # end
    end
<commit_msg>building rdocs<commit_after>#!/usr/bin/ruby

require 'fileutils'
require 'net/ftp'
require 'open3'
require 'pp'
require 'rubygems'
require 'rake'
require 'progress'

class String
  def /(s)
    File.join(self, s)
  end
end

BASE_PATH = File.expand_path(File.dirname(__FILE__))
DOCS_PATH = BASE_PATH / 'docs'
SOURSES_PATH = BASE_PATH / 'sources'

class RdocAll
  class << self
    def update_sources(options = {})
      Base.update_all_sources(options)
    end

    def build_documentation(options = {})
      Base.build_documentation(options)
    end
  end
end

class RdocAll::Base
  class << self
    def inherited(subclass)
      (@subclasses ||= []) << subclass
    end

    def update_all_sources(options = {})
      Dir.chdir(SOURSES_PATH) do
        @subclasses.each do |subclass|
          subclass.update_sources(options)
        end
      end
    end

    def build_documentation(options = {})
      Dir.chdir(SOURSES_PATH) do
        @@rdoc_tasks = []

        @subclasses.each do |subclass|
          subclass.add_rdoc_tasks
        end

        @@rdoc_tasks.each_with_progress('Building docs') do |rdoc_task|
          File.open('.document', 'w') do |f|
            if rdoc_task[:source_pathes].empty?
              f.puts(rdoc_task[:base_path])
            else
              rdoc_task[:source_pathes].each do |source_path|
                f.puts(rdoc_task[:base_path] / source_path)
              end
            end
          end

          doc_path = DOCS_PATH / rdoc_task[:base_path]
          remove_if_present(doc_path) if Dir[doc_path / '*'].empty? || options[:force]
          cmd = %w(hanna)
          cmd << '-U' if options[:force]
          cmd << '-o' << doc_path
          cmd << '-t' << rdoc_task[:title]

          system(*cmd)
        end
      end
    end

  protected

    def add_rdoc_task(base_path, source_pathes = [])
      @@rdoc_tasks << {:base_path => base_path, :source_pathes => source_pathes, :title => base_path.sub('s/', ' — ')}
    end

{'fraction_non_alphanumeric': 0.12296511627906977, 'fraction_numerical': 0.0005813953488372093, 'mean_word_length': 2.4934010152284265, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2297385', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1169, 'n_words': 241}