<commit_before>

class Vocco::Generator

  require 'vocco/generator/css'

  attr_reader :globs, :notes, :out, :name, :site

  def initialize(opts)
    @globs  = opts[:files].map {|path| path.sub(/\/$/, '') }
    @out    = opts[:out]
    @notes  = opts[:notes]
    @name   = opts[:name].capitalize
    @site   = opts[:site]
    @vim    = opts[:vim]
    validate_out
  end

  def run
    run_vim; postprocess
  end
  alias :run! :run

  def trimmed_globs
    @trimmed_globs ||= @globs.map do |glob|
      glob.gsub(/(^\.\/)|(\/$)/, '')
    end
  end

  def scopes
    @scopes ||= trimmed_globs.map do |glob|
      tokens = glob.split('*')
      tokens.size > 1 ? tokens.first : nil
    end.compact
  end

  def glob_regex
    @glob_regex ||= begin
      str = scopes.map do |scope|
        Regexp::escape(scope)
      end.join('|')
      /^(\.\/)?(#{str})(\/)?/
    end
  end

  def doc_regex
    @doc_regex ||= begin
      str = Regexp::escape(@out) + '/'
      /^#{str}/
    end
  end

  def files
    @files ||= @globs.map do |glob|
      Dir[glob]
    end.flatten.uniq.map do |path|
      SourceFile.new(path, self)
    end
  end

  private

    def validate_out
      unless File.directory?(@out)
        raise "#{@out} is not a valid directory."
      end
    end

    def run_vim
      script = Tempfile.new('vimdocco')
      script.write(vimscript)
      script.close
      puts 'Running ' + vim_command + ':'
      system "#{vim_command} -f -S #{script.path}"
      script.delete
    end

    def vim_command
      @vim_command ||= @vim.detect do |c|
        `which #{c}`.chomp.strip != ''
      end || abort("You must install one of #{@vim.inspect}.")
    end

    def vimscript
      @vimscript ||= begin
        s = "let html_no_pre = 1\n"
        files.each do |file|
          s << "view! #{file.file}\n"
          s << "TOhtml\n"
          s << "w! #{file.doc_path}\n"
        end
        s << ":qall!\n"
      end
    end

    def postprocess
      hpricotify_docs do |doc, file|
        doc.at('style').after('<style type="text/css">' + CSS + '</style>')
        doc.at('title').inner_html = name + ' : ' + file.basename
        inner = doc.at('body').inner_html
        doc.at('body').swap "<body><div class='code'>#{inner}</div></body>"
        doc.at('.code').after file.render_template
      end
    end

    def hpricotify_docs
      files.each do |file|
        doc = File.open(file.doc_path) {|f| Hpricot(f) }
        yield(doc, file)
        File.open(file.doc_path, 'w')  {|f| f.write(doc.to_html) }
      end
    end
end

class Vocco::Generator::SourceFile
  require 'vocco/generator/source_file/html_template'

  def initialize(file, generator)
    @file       = file
    @gen        = generator
  end

  attr_reader :file

  def short_path
    @short_path ||= @file.sub(@gen.glob_regex, '')
  end

  def short_dirname
    File.dirname(short_path).sub(/^\.\//, '')
  end

  def dirname
    File.dirname(@file)
  end

  def basename
    File.basename(@file)
  end

  def doc_prefix
    short_dirname == '.' ? '' : short_dirname.gsub('/', '-') + '-'
  end

  def doc_path
    File.join(@gen.out, doc_prefix + basename + '.html')
  end

  def doc_link
    './' + doc_path.sub(@gen.doc_regex, '')
  end

  def notes
    rendered = render_notes
    rendered.any? ?
      rendered.join("\n") : nil
  end

  def render_notes
    Dir[note_glob].map do |path|
      Tilt.new(path).render
    end
  end

  def note_glob
    dirs = [dirname]
    @gen.notes.each do |dir|
      dirs << File.join(dir, short_dirname)
    end
    ext       = '.{textile,md,mkd,markdowb,rdoc}'
    dir_glob  = '{' + dirs.join(',') + '}/'
    dir_glob + basename + ext
  end

  class_eval <<-EOF, '(template)'
    def render_template
      #{HTML_TEMPLATE}
    end
  EOF
end
<commit_msg>bug fix: typo prevented .markdown notes from being included.<commit_after>

class Vocco::Generator

  require 'vocco/generator/css'

  attr_reader :globs, :notes, :out, :name, :site

  def initialize(opts)
    @globs  = opts[:files].map {|path| path.sub(/\/$/, '') }
    @out    = opts[:out]
    @notes  = opts[:notes]
    @name   = opts[:name].capitalize
    @site   = opts[:site]
    @vim    = opts[:vim]
    validate_out
  end

  def run
    run_vim; postprocess
  end
  alias :run! :run

  def trimmed_globs
    @trimmed_globs ||= @globs.map do |glob|
      glob.gsub(/(^\.\/)|(\/$)/, '')
    end
  end

  def scopes
    @scopes ||= trimmed_globs.map do |glob|
      tokens = glob.split('*')
      tokens.size > 1 ? tokens.first : nil
    end.compact
  end

  def glob_regex
    @glob_regex ||= begin
      str = scopes.map do |scope|
        Regexp::escape(scope)
      end.join('|')
      /^(\.\/)?(#{str})(\/)?/
    end
  end

  def doc_regex
    @doc_regex ||= begin
      str = Regexp::escape(@out) + '/'
      /^#{str}/
    end
  end

  def files
    @files ||= @globs.map do |glob|
      Dir[glob]
    end.flatten.uniq.map do |path|
      SourceFile.new(path, self)
    end
  end

  private

    def validate_out
      unless File.directory?(@out)
        raise "#{@out} is not a valid directory."
      end
    end

    def run_vim
      script = Tempfile.new('vimdocco')
      script.write(vimscript)
      script.close
      puts 'Running ' + vim_command + ':'
      system "#{vim_command} -f -S #{script.path}"
      script.delete
    end

    def vim_command
      @vim_command ||= @vim.detect do |c|
        `which #{c}`.chomp.strip != ''
      end || abort("You must install one of #{@vim.inspect}.")
    end

    def vimscript
      @vimscript ||= begin
        s = "let html_no_pre = 1\n"
        files.each do |file|
          s << "view! #{file.file}\n"
          s << "TOhtml\n"
          s << "w! #{file.doc_path}\n"
        end
        s << ":qall!\n"
      end
    end

    def postprocess
      hpricotify_docs do |doc, file|
        doc.at('style').after('<style type="text/css">' + CSS + '</style>')
        doc.at('title').inner_html = name + ' : ' + file.basename
        inner = doc.at('body').inner_html
        doc.at('body').swap "<body><div class='code'>#{inner}</div></body>"
        doc.at('.code').after file.render_template
      end
    end

    def hpricotify_docs
      files.each do |file|
        doc = File.open(file.doc_path) {|f| Hpricot(f) }
        yield(doc, file)
        File.open(file.doc_path, 'w')  {|f| f.write(doc.to_html) }
      end
    end
end

class Vocco::Generator::SourceFile
  require 'vocco/generator/source_file/html_template'

  def initialize(file, generator)
    @file       = file
    @gen        = generator
  end

  attr_reader :file

  def short_path
    @short_path ||= @file.sub(@gen.glob_regex, '')
  end

  def short_dirname
    File.dirname(short_path).sub(/^\.\//, '')
  end

  def dirname
    File.dirname(@file)
  end

  def basename
    File.basename(@file)
  end

  def doc_prefix
    short_dirname == '.' ? '' : short_dirname.gsub('/', '-') + '-'
  end

  def doc_path
    File.join(@gen.out, doc_prefix + basename + '.html')
  end

  def doc_link
    './' + doc_path.sub(@gen.doc_regex, '')
  end

  def notes
    rendered = render_notes
    rendered.any? ?
      rendered.join("\n") : nil
  end

  def render_notes
    Dir[note_glob].map do |path|
      Tilt.new(path).render
    end
  end

  def note_glob
    dirs = [dirname]
    @gen.notes.each do |dir|
      dirs << File.join(dir, short_dirname)
    end
    ext       = '.{textile,md,mkd,markdown,rdoc}'
    dir_glob  = '{' + dirs.join(',') + '}/'
    dir_glob + basename + ext
  end

  class_eval <<-EOF, '(template)'
    def render_template
      #{HTML_TEMPLATE}
    end
  EOF
end

{'fraction_non_alphanumeric': 0.1573550677008019, 'fraction_numerical': 0.0005258314710135402, 'mean_word_length': 2.1660424469413235, 'pattern_counts': {'":': 2, '<': 39, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '317770', 'n_tokens_mistral': 2825, 'n_tokens_neox': 2718, 'n_words': 602}