<commit_before>#TODO: This should probably be changed to use Sinatra::Base so that Object is not poulted with Sinatra methods...
require 'sinatra'
require 'sinatra/contrib'
require 'sequel'

module Crossbeams
  module DataminerPortal

    class ConfigMerger
      # Write a new config file by applying the client-specific settings over the defaults.
      def self.merge_config_files(base_file, over_file, new_config_file_name)
        f = File.open(new_config_file_name, 'w')
        YAML.dump(YAML.load_file(base_file).merge(YAML.load_file(over_file)), f)
        f.close

        hold = File.readlines(new_config_file_name)[1..-1].join
        File.open(new_config_file_name,"w") {|fw| fw.write(hold) }
      end
    end

    class WebPortal < Sinatra::Application

      configure do
        enable :logging
        # mkdir log if it does not exist...
        Dir.mkdir('log') unless Dir.exist?('log')
        file = File.new("log/dm_#{settings.environment}.log", 'a+')
        file.sync = true
        use Rack::CommonLogger, file

        set :environment, :production
        set :root, File.dirname(__FILE__)
        # :method_override - use for PATCH etc forms? http://www.rubydoc.info/github/rack/rack/Rack/MethodOverride
        set :app_file, __FILE__
        # :raise_errors - should be set so that Rack::ShowExceptions or the server can be used to handle the error...
        enable :show_exceptions # because we are forcing the environment to production...
        set :appname, 'tst'
        set :url_prefix, ENV['DM_PREFIX']  ? "#{ENV['DM_PREFIX']}/" : ''
        set :protection, except: :frame_options # so it can be loaded in another app's iframe...


        set :base_file, "#{FileUtils.pwd}/config/dm_defaults.yml"
        set :over_file, "#{FileUtils.pwd}/config/dm_#{ENV['DM_CLIENT'] || 'defaults'}.yml"
        set :new_config_file_name, "#{FileUtils.pwd}/config/dm_config_file.yml" # This could be a StringIO...
      end

      if settings.base_file == settings.over_file
        FileUtils.cp(settings.base_file, settings.new_config_file_name)
      else
        ConfigMerger.merge_config_files(settings.base_file, settings.over_file, settings.new_config_file_name)
      end
      config_file settings.new_config_file_name

      # TODO: Need to see how this should be done when running under passenger/thin/puma...
      DB = Sequel.postgres(settings.database['name'], :user => settings.database['user'], :password => settings.database['password'], :host => settings.database['host'] || 'localhost')


      def sql_to_highlight(sql)
        # wrap sql @ 120
        width = 120
        ar = sql.gsub(/from /i, "\nFROM ").gsub(/where /i, "\nWHERE ").gsub(/(left outer join |left join |inner join |join )/i, "\n\\1").split("\n")
        wrapped_sql = ar.map {|a| a.scan(/\S.{0,#{width-2}}\S(?=\s|$)|\S+/).join("\n") }.join("\n")
        # wrapped_sql = sql.scan(/\S.{0,#{width-2}}\S(?=\s|$)|\S+/).join("\n")
        # opts = {:css_class    => nil,
        #         :inline_theme => 'github',
        #         :line_numbers => false}

        theme = Rouge::Themes::Github.new
        formatter = Rouge::Formatters::HTMLInline.new(theme)
        #lexer  = Rouge::Lexer.find('sql')
        lexer  = Rouge::Lexers::SQL.new
        formatter.format(lexer.lex(wrapped_sql))
      end

      def lookup_report(id)
        DmReportLister.new(settings.dm_reports_location).get_report_by_id(id)
      end

      def clean_where(sql)
        rems = sql.scan( /\{(.+?)\}/).flatten.map {|s| "#{s}={#{s}}" }
        rems.each {|r| sql.gsub!(%r|and\s+#{r}|i,'') }
        rems.each {|r| sql.gsub!(r,'') }
        sql.sub!(/where\s*\(\s+\)/i, '')
        sql
      end


      # ERB helpers.
      helpers do
        # def h(text)
        #   Rack::Utils.escape_html(text)
        # end
        def make_options(ar)
          ar.map do |a|
            if a.kind_of?(Array)
              "<option value=\"#{a.last}\">#{a.first}</option>"
            else
              "<option value=\"#{a}\">#{a}</option>"
            end
          end.join("\n")
        end

        def the_url_prefix
          settings.url_prefix
        end

        def menu
          "<p><a href='/#{settings.url_prefix}index'>Return to report index</a></p>"
        end
      end


      get '/' do
        # dataset = DB['select id from users']
        # "GOT THERE... running with #{settings.appname} <a href='#{settings.url_prefix}test_page'>Go to test page</a><p>Users: #{dataset.count} with ids: #{dataset.map(:id).join(', ')}.</p><p>Random user: #{DB['select user_name FROM users LIMIT 1'].first[:user_name]}</p>"
        "<a href='/#{settings.url_prefix}index'>DATAMINER REPORT INDEX</a>"
      end

      get '/index' do
        # TODO: sort report list, group, add tags etc...

        rpt_list = DmReportLister.new(settings.dm_reports_location).get_report_list(true)<commit_msg>Fix call to report list.<commit_after>#TODO: This should probably be changed to use Sinatra::Base so that Object is not poulted with Sinatra methods...
require 'sinatra'
require 'sinatra/contrib'
require 'sequel'

module Crossbeams
  module DataminerPortal

    class ConfigMerger
      # Write a new config file by applying the client-specific settings over the defaults.
      def self.merge_config_files(base_file, over_file, new_config_file_name)
        f = File.open(new_config_file_name, 'w')
        YAML.dump(YAML.load_file(base_file).merge(YAML.load_file(over_file)), f)
        f.close

        hold = File.readlines(new_config_file_name)[1..-1].join
        File.open(new_config_file_name,"w") {|fw| fw.write(hold) }
      end
    end

    class WebPortal < Sinatra::Application

      configure do
        enable :logging
        # mkdir log if it does not exist...
        Dir.mkdir('log') unless Dir.exist?('log')
        file = File.new("log/dm_#{settings.environment}.log", 'a+')
        file.sync = true
        use Rack::CommonLogger, file

        set :environment, :production
        set :root, File.dirname(__FILE__)
        # :method_override - use for PATCH etc forms? http://www.rubydoc.info/github/rack/rack/Rack/MethodOverride
        set :app_file, __FILE__
        # :raise_errors - should be set so that Rack::ShowExceptions or the server can be used to handle the error...
        enable :show_exceptions # because we are forcing the environment to production...
        set :appname, 'tst'
        set :url_prefix, ENV['DM_PREFIX']  ? "#{ENV['DM_PREFIX']}/" : ''
        set :protection, except: :frame_options # so it can be loaded in another app's iframe...


        set :base_file, "#{FileUtils.pwd}/config/dm_defaults.yml"
        set :over_file, "#{FileUtils.pwd}/config/dm_#{ENV['DM_CLIENT'] || 'defaults'}.yml"
        set :new_config_file_name, "#{FileUtils.pwd}/config/dm_config_file.yml" # This could be a StringIO...
      end

      if settings.base_file == settings.over_file
        FileUtils.cp(settings.base_file, settings.new_config_file_name)
      else
        ConfigMerger.merge_config_files(settings.base_file, settings.over_file, settings.new_config_file_name)
      end
      config_file settings.new_config_file_name

      # TODO: Need to see how this should be done when running under passenger/thin/puma...
      DB = Sequel.postgres(settings.database['name'], :user => settings.database['user'], :password => settings.database['password'], :host => settings.database['host'] || 'localhost')


      def sql_to_highlight(sql)
        # wrap sql @ 120
        width = 120
        ar = sql.gsub(/from /i, "\nFROM ").gsub(/where /i, "\nWHERE ").gsub(/(left outer join |left join |inner join |join )/i, "\n\\1").split("\n")
        wrapped_sql = ar.map {|a| a.scan(/\S.{0,#{width-2}}\S(?=\s|$)|\S+/).join("\n") }.join("\n")
        # wrapped_sql = sql.scan(/\S.{0,#{width-2}}\S(?=\s|$)|\S+/).join("\n")
        # opts = {:css_class    => nil,
        #         :inline_theme => 'github',
        #         :line_numbers => false}

        theme = Rouge::Themes::Github.new
        formatter = Rouge::Formatters::HTMLInline.new(theme)
        #lexer  = Rouge::Lexer.find('sql')
        lexer  = Rouge::Lexers::SQL.new
        formatter.format(lexer.lex(wrapped_sql))
      end

      def lookup_report(id)
        DmReportLister.new(settings.dm_reports_location).get_report_by_id(id)
      end

      def clean_where(sql)
        rems = sql.scan( /\{(.+?)\}/).flatten.map {|s| "#{s}={#{s}}" }
        rems.each {|r| sql.gsub!(%r|and\s+#{r}|i,'') }
        rems.each {|r| sql.gsub!(r,'') }
        sql.sub!(/where\s*\(\s+\)/i, '')
        sql
      end


      # ERB helpers.
      helpers do
        # def h(text)
        #   Rack::Utils.escape_html(text)
        # end
        def make_options(ar)
          ar.map do |a|
            if a.kind_of?(Array)
              "<option value=\"#{a.last}\">#{a.first}</option>"
            else
              "<option value=\"#{a}\">#{a}</option>"
            end
          end.join("\n")
        end

        def the_url_prefix
          settings.url_prefix
        end

        def menu
          "<p><a href='/#{settings.url_prefix}index'>Return to report index</a></p>"
        end
      end


      get '/' do
        # dataset = DB['select id from users']
        # "GOT THERE... running with #{settings.appname} <a href='#{settings.url_prefix}test_page'>Go to test page</a><p>Users: #{dataset.count} with ids: #{dataset.map(:id).join(', ')}.</p><p>Random user: #{DB['select user_name FROM users LIMIT 1'].first[:user_name]}</p>"
        "<a href='/#{settings.url_prefix}index'>DATAMINER REPORT INDEX</a>"
      end

      get '/index' do
        # TODO: sort report list, group, add tags etc...

        rpt_list = DmReportLister.new(settings.dm_reports_location).get_report_list(persist: true)
{'fraction_non_alphanumeric': 0.1542237676416445, 'fraction_numerical': 0.0028635712824708528, 'mean_word_length': 2.549546279491833, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6809751', 'n_tokens_mistral': 3266, 'n_tokens_neox': 3172, 'n_words': 702}