<commit_before>require 'fileutils'
require 'brahma/utils/string_helper'

module Brahma
  module Utils
    module WikiHelper
      WIKI_ROOT="#{Rails.root}/tmp/wiki"
      module_function

      def update(url)
        #todo 防注入
        unless Dir.exist?(WIKI_ROOT)
          FileUtils.mkpath WIKI_ROOT
        end
        base = "#{WIKI_ROOT}/#{Brahma::Utils::StringHelper.md5 url}"
        Dir.exist?("#{base}/.git") ? `cd #{base} && git pull` : `git clone #{url} #{base}`
      end

      def get(name)
        name = "#{WIKI_ROOT}/#{name}.md"
        if File.file?(name)
          File.open(name, 'r') { |f| Brahma::Utils::StringHelper.md2html f.read }
        end
      end

      def each(url=nil)
        len = WIKI_ROOT.size
        Dir.glob("#{WIKI_ROOT}#{url ? "/#{url}": ''}/**/*.md").each do |fn|
          name = fn[len+1, fn.size-len-4]
          yield name
        end
      end
    end
  end
end<commit_msg>fix wiki bug<commit_after>require 'fileutils'
require 'brahma/utils/string_helper'

module Brahma
  module Utils
    module WikiHelper
      WIKI_ROOT="#{Rails.root}/tmp/wiki"
      module_function

      def update(url)
        #todo 防注入
        # unless Dir.exist?(WIKI_ROOT)
        #   FileUtils.mkpath WIKI_ROOT
        # end
        base = "#{WIKI_ROOT}/#{Brahma::Utils::StringHelper.md5 url}"
        Dir.exist?("#{base}/.git") ? `cd #{base} && git pull` : `git clone #{url} #{base}`
      end

      def get(name)
        name = "#{WIKI_ROOT}/#{name}.md"
        if File.file?(name)
          File.open(name, 'r') { |f| Brahma::Utils::StringHelper.md2html f.read }
        end
      end

      def each(url=nil)
        len = WIKI_ROOT.size
        Dir.glob("#{WIKI_ROOT}#{url ? "/#{url}": ''}/**/*.md").each do |fn|
          name = fn[len+1, fn.size-len-4]
          yield name
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.1584640346133045, 'fraction_numerical': 0.004326663061114116, 'mean_word_length': 2.1040268456375837, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '322927', 'n_tokens_mistral': 682, 'n_tokens_neox': 674, 'n_words': 133}