<commit_before>require 'sinatra'
require 'json'
require './lib/slack_import'
require './lib/slack'
require './lib/db'

config = YAML.load_file('./config.yml')

configure do
  set :absolute_redirects, false
  set :prefixed_redirects, true
end

def users
  hashed_users = {}
  Users.find.each do |u|
    hashed_users[u[:id]] = u
  end
  hashed_users
end

def channels
  hashed_channels = {}
  Channels.find.sort(name: 1).each do |c|
    hashed_channels[c[:id]] = c
  end
  hashed_channels
end

def ims
  hashed_users = users
  hashed_ims = {}
  Ims.find.each do |c|
    hashed_ims[c[:id]] = c
    hashed_ims[c[:id]][:name] = hashed_users[c[:user]][:name]
  end
  hashed_ims.sort_by {|k, v| v[:name] }.to_h
end

def messages(params)
  limit = params[:limit] || 100
  condition = {
    ts: { '$lt' =>  params[:min_ts] || Time.now.to_i.to_s },
    hidden: { '$ne' => true }
  }
  condition[:channel] = params[:channel] unless params[:channel].nil?
  condition['$or'] = [
    # normal message
    { text: Regexp.new(params[:search]) },
    # bot message
    {
      attachments: {
        '$elemMatch' => { text: Regexp.new(params[:search]) }
      },
      subtype: 'bot_message'
    }
  ] unless params[:search].nil?

  all_messages = Messages
    .find(condition)
    .sort(ts: -1)
    .limit(limit + 1)
  has_more_message = all_messages.count > limit

  return all_messages.limit(limit), has_more_message
end

get '/users.json' do
  content_type :json
  users.to_json
end

get '/channels.json' do
  content_type :json
  channels.to_json
end

get '/ims.json' do
  content_type :json
  ims.to_json
end

post '/messages/:channel.json' do
  all_messages, has_more_message = messages(
    channel: params[:channel],
    min_ts: params[:min_ts]
  )

  content_type :json
  {
    messages: all_messages.to_a.reverse,
    has_more_message: has_more_message
  }.to_json
end

get '/team.json' do
  content_type :json
  # TODO: cache in redis or mongodb or in memory?
  Slack.team_info['team'].to_json
end

post '/import_backup' do
  exported_file = '/tmp/slack_export.zip'
  FileUtils.move(params[:file][:tempfile], exported_file)
  SlackImport.new.import_from_file(exported_file)
end<commit_msg>comment<commit_after>require 'sinatra'
require 'json'
require './lib/slack_import'
require './lib/slack'
require './lib/db'

config = YAML.load_file('./config.yml')

configure do
  set :absolute_redirects, false
  set :prefixed_redirects, true
end

def users
  hashed_users = {}
  Users.find.each do |u|
    hashed_users[u[:id]] = u
  end
  hashed_users
end

def channels
  hashed_channels = {}
  Channels.find.sort(name: 1).each do |c|
    hashed_channels[c[:id]] = c
  end
  hashed_channels
end

def ims
  hashed_users = users
  hashed_ims = {}
  Ims.find.each do |c|
    hashed_ims[c[:id]] = c
    hashed_ims[c[:id]][:name] = hashed_users[c[:user]][:name]
  end
  hashed_ims.sort_by {|k, v| v[:name] }.to_h
end

def messages(params)
  limit = params[:limit] || 100
  condition = {
    ts: { '$lt' =>  params[:min_ts] || Time.now.to_i.to_s },
    hidden: { '$ne' => true }
  }
  condition[:channel] = params[:channel] unless params[:channel].nil?
  condition['$or'] = [
    # normal message
    { text: Regexp.new(params[:search]) },
    # bot message
    {
      attachments: {
        '$elemMatch' => { text: Regexp.new(params[:search]) }
      },
      subtype: 'bot_message'
    }
  ] unless params[:search].nil?

  all_messages = Messages
    .find(condition)
    .sort(ts: -1)
    .limit(limit + 1)
  has_more_message = all_messages.count > limit

  return all_messages.limit(limit), has_more_message
end

get '/users.json' do
  content_type :json
  users.to_json
end

get '/channels.json' do
  content_type :json
  channels.to_json
end

get '/ims.json' do
  content_type :json
  ims.to_json
end

post '/messages/:channel.json' do
  all_messages, has_more_message = messages(
    channel: params[:channel],
    min_ts: params[:min_ts]
  )

  content_type :json
  {
    messages: all_messages.to_a.reverse,
    has_more_message: has_more_message
  }.to_json
end

get '/team.json' do
  content_type :json
  # TODO: cache in redis or mongodb or in memory?
  Slack.team_info['team'].to_json
end

post '/import_backup' do
  exported_file = '/tmp/slack_export.zip'
  FileUtils.move(params[:file][:tempfile], exported_file)
  # TODO: show progress when import
  SlackImport.new.import_from_file(exported_file)
end
{'fraction_non_alphanumeric': 0.13915416098226466, 'fraction_numerical': 0.002728512960436562, 'mean_word_length': 3.5680166147455865, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '62404', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1639, 'n_words': 350}