<commit_before>require 'tempfile'

class Vimdb::Keys < Vimdb::Item
  class << self; attr_accessor :config end
  self.config = {
    plugins_dir: 'bundle',
    modifiers: {'<Esc>' => 'E'},
    mode_map: {'!' => 'ci', 'v' => 'vs', 'x' => 'v', 'l' => 'ci'},
  }

  def initialize
    @modifiers, @mode_map = self.class.config.values_at(:modifiers, :mode_map)
    @plugins_dir = self.class.config[:plugins_dir]
  end

  def create
    keys = parse_index_file create_index_file
    @leader ||= get_leader
    @modifiers[@leader] ||= 'L'
    keys + parse_map_file(create_map_file)
  end

  def search(keys, query, options = {})
    keys = super
    if options[:mode]
      keys.select! do |key|
        options[:mode].split('').any? {|m| key[:mode].include?(m) }
      end
    end
    keys
  end

  def info
    "Created using index.txt and :map"
  end

  def display_fields
    [:key, :mode, :from, :desc]
  end

  private

  def get_leader
    file = Tempfile.new('vim-leader').path
    leader_cmd = %[silent! echo exists("mapleader") ? mapleader : ""]
    vim "redir! > #{file}", leader_cmd, 'redir END'
    leader = File.readlines(file).last.chomp
    {' ' => '<Space>', '' => '\\'}[leader] || leader
  end

  def create_index_file
    file = Tempfile.new('vim-index').path
    vim 'silent help index.txt', "silent! w! #{file}"
    file
  end

  def parse_index_file(file)
    lines = File.read(file).split("\n")
    sections = lines.slice_before(/^={10,}/).to_a
    header_modes = [
      ['1. Insert mode', 'i'], ['2.1', 'ovs'], ['2.', 'n'],
      ['3. Visual mode', 'vs'], ['4. Command-line editing', 'c']
    ]

    keys = []
    # skip intro and last Ex section
    sections[1..-2].each do |section_lines|
      mode = header_modes.find {|k,v|
        section_lines[1] =~ Regexp.new('^' + Regexp.quote(k))
      }.to_a[1] || '?'

      #drop section header
      section_lines = section_lines.drop_while {|e| e !~ /^\|/ }

      section_lines.each do |e|
        cols = e.split(/\t+/)
        if cols.size >= 3
          key = translate_index_key cols[-2]
          keys << {mode: mode, key: key, desc: cols[-1].strip, :from => 'default'}
        # add desc from following lines
        elsif cols.size == 2 && cols[0] == ''
          keys[-1][:desc] += ' ' + cols[1].strip
        # else
        # TODO: parse few edge cases
        end
      end
    end
    keys
  end

  def translate_index_key(key)
    key.gsub(/CTRL-([A-Z])/) {|s| "C-#{$1.downcase}" }
  end

  def create_map_file
    file = Tempfile.new('vim-map').path
    vim "redir! > #{file}", "silent! verbose map", "silent! verbose map!",
      'redir END'
    file
  end

  def parse_map_file(file)
    lines = File.read(file).strip.split("\n")
    lines.slice_before {|e| e !~ /Last set/ }.map do |arr|
      key = {}

      key[:file] = arr[1].to_s[%r{Last set from (\S+)}, 1] or next
      key[:from] = key[:file].to_s[%r{/#{@plugins_dir}/([^/]+)\S+}, 1] || 'user'
      key[:from] += ' plugin' if key[:from] != 'user'

      key[:key]  = arr[0][/^\S*\s+(\S+)/, 1]
      next if key[:key][/^(<Plug>|<SNR>)/]
      key[:key] = translate_map_key(key[:key])

      key[:desc] = arr[0][/^\S*\s+\S+\s+(.*)$/, 1]<commit_msg>Convert all CTRL index keys to C<commit_after>require 'tempfile'

class Vimdb::Keys < Vimdb::Item
  class << self; attr_accessor :config end
  self.config = {
    plugins_dir: 'bundle',
    modifiers: {'<Esc>' => 'E'},
    mode_map: {'!' => 'ci', 'v' => 'vs', 'x' => 'v', 'l' => 'ci'},
  }

  def initialize
    @modifiers, @mode_map = self.class.config.values_at(:modifiers, :mode_map)
    @plugins_dir = self.class.config[:plugins_dir]
  end

  def create
    keys = parse_index_file create_index_file
    @leader ||= get_leader
    @modifiers[@leader] ||= 'L'
    keys + parse_map_file(create_map_file)
  end

  def search(keys, query, options = {})
    keys = super
    if options[:mode]
      keys.select! do |key|
        options[:mode].split('').any? {|m| key[:mode].include?(m) }
      end
    end
    keys
  end

  def info
    "Created using index.txt and :map"
  end

  def display_fields
    [:key, :mode, :from, :desc]
  end

  private

  def get_leader
    file = Tempfile.new('vim-leader').path
    leader_cmd = %[silent! echo exists("mapleader") ? mapleader : ""]
    vim "redir! > #{file}", leader_cmd, 'redir END'
    leader = File.readlines(file).last.chomp
    {' ' => '<Space>', '' => '\\'}[leader] || leader
  end

  def create_index_file
    file = Tempfile.new('vim-index').path
    vim 'silent help index.txt', "silent! w! #{file}"
    file
  end

  def parse_index_file(file)
    lines = File.read(file).split("\n")
    sections = lines.slice_before(/^={10,}/).to_a
    header_modes = [
      ['1. Insert mode', 'i'], ['2.1', 'ovs'], ['2.', 'n'],
      ['3. Visual mode', 'vs'], ['4. Command-line editing', 'c']
    ]

    keys = []
    # skip intro and last Ex section
    sections[1..-2].each do |section_lines|
      mode = header_modes.find {|k,v|
        section_lines[1] =~ Regexp.new('^' + Regexp.quote(k))
      }.to_a[1] || '?'

      #drop section header
      section_lines = section_lines.drop_while {|e| e !~ /^\|/ }

      section_lines.each do |e|
        cols = e.split(/\t+/)
        if cols.size >= 3
          key = translate_index_key cols[-2]
          keys << {mode: mode, key: key, desc: cols[-1].strip, :from => 'default'}
        # add desc from following lines
        elsif cols.size == 2 && cols[0] == ''
          keys[-1][:desc] += ' ' + cols[1].strip
        # else
        # TODO: parse few edge cases
        end
      end
    end
    keys
  end

  def translate_index_key(key)
    key.gsub(/CTRL-(\S)/) {|s| "C-#{$1.downcase}" }
  end

  def create_map_file
    file = Tempfile.new('vim-map').path
    vim "redir! > #{file}", "silent! verbose map", "silent! verbose map!",
      'redir END'
    file
  end

  def parse_map_file(file)
    lines = File.read(file).strip.split("\n")
    lines.slice_before {|e| e !~ /Last set/ }.map do |arr|
      key = {}

      key[:file] = arr[1].to_s[%r{Last set from (\S+)}, 1] or next
      key[:from] = key[:file].to_s[%r{/#{@plugins_dir}/([^/]+)\S+}, 1] || 'user'
      key[:from] += ' plugin' if key[:from] != 'user'

      key[:key]  = arr[0][/^\S*\s+(\S+)/, 1]
      next if key[:key][/^(<Plug>|<SNR>)/]
      key[:key] = translate_map_key(key[:key])

      key[:desc] = arr[0][/^\S*\s+\S+\s+(.*)$/, 1]
{'fraction_non_alphanumeric': 0.19899874843554444, 'fraction_numerical': 0.008448060075093867, 'mean_word_length': 2.593591905564924, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3780016', 'n_tokens_mistral': 2464, 'n_tokens_neox': 2365, 'n_words': 537}