<commit_before>/*
 * IdeaVim - Vim emulator for IDEs based on the IntelliJ platform
 * Copyright (C) 2003-2021 The IdeaVim authors
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

package com.maddyhome.idea.vim.extension.matchit

import com.intellij.openapi.actionSystem.DataContext
import com.intellij.openapi.editor.Caret
import com.intellij.openapi.editor.Editor
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.PsiComment
import com.intellij.psi.PsiElement
import com.intellij.psi.util.PsiTreeUtil
import com.intellij.psi.util.elementType
import com.maddyhome.idea.vim.VimPlugin
import com.maddyhome.idea.vim.command.Argument
import com.maddyhome.idea.vim.command.Command
import com.maddyhome.idea.vim.command.CommandFlags
import com.maddyhome.idea.vim.command.CommandState.Companion.getInstance
import com.maddyhome.idea.vim.command.MappingMode
import com.maddyhome.idea.vim.command.MotionType
import com.maddyhome.idea.vim.command.OperatorArguments
import com.maddyhome.idea.vim.extension.VimExtension
import com.maddyhome.idea.vim.extension.VimExtensionFacade
import com.maddyhome.idea.vim.extension.VimExtensionFacade.putKeyMappingIfMissing
import com.maddyhome.idea.vim.extension.VimExtensionHandler
import com.maddyhome.idea.vim.group.MotionGroup
import com.maddyhome.idea.vim.handler.Motion
import com.maddyhome.idea.vim.handler.MotionActionHandler
import com.maddyhome.idea.vim.handler.toMotionOrError
import com.maddyhome.idea.vim.helper.EditorHelper
import com.maddyhome.idea.vim.helper.PsiHelper
import com.maddyhome.idea.vim.helper.StringHelper.parseKeys
import com.maddyhome.idea.vim.helper.commandState
import com.maddyhome.idea.vim.helper.enumSetOf
import com.maddyhome.idea.vim.helper.getTopLevelEditor
import com.maddyhome.idea.vim.helper.vimForEachCaret
import java.util.*
import java.util.regex.Pattern

/**
 * Port of matchit.vim (https://github.com/chrisbra/matchit)
 * Languages currently supported:
 *   Ruby, Embedded Ruby, XML, and HTML (including HTML in JavaScript/TypeScript, JSX/TSX, and JSPs)
 *
 * @author <NAME> (@myzeiri)
 */
class Matchit : VimExtension {

  override fun getName(): String = "matchit"

  override fun init() {
    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(MatchitMotion)"), owner, MatchitHandler(false), false)
    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(MatchitMotion)"), owner, MatchitHandler(false), false)
    putKeyMappingIfMissing(MappingMode.NXO, parseKeys("%"), owner, parseKeys("<Plug>(MatchitMotion)"), true)

    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(ReverseMatchitMotion)"), owner, MatchitHandler(true), false)
    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(ReverseMatchitMotion)"), owner, MatchitHandler(true), false)
    putKeyMappingIfMissing(MappingMode.NXO, parseKeys("g%"), owner, parseKeys("<Plug>(ReverseMatchitMotion)"), true)
  }

  private class MatchitAction : MotionActionHandler.ForEachCaret() {
    var reverse = false
    var isInOpPending = false

    override val flags: EnumSet<CommandFlags> = enumSetOf(CommandFlags.FLAG_SAVE_JUMP)

    override fun getOffset(
      editor: Editor,
      caret: Caret,
      context: DataContext,
      argument: Argument?,
      operatorArguments: OperatorArguments,
    ): Motion {
      return getMatchitOffset(editor, caret, operatorArguments.count0, isInOpPending, reverse).toMotionOrError()
    }

    override fun process(cmd: Command) {
      motionType = if (cmd.rawCount != 0) MotionType.LINE_WISE else MotionType.INCLUSIVE
    }

    override var motionType: MotionType = MotionType.INCLUSIVE
  }

  private class MatchitHandler(private val reverse: Boolean) : VimExtensionHandler {

    override fun execute(editor: Editor, context: DataContext) {
      val commandState = getInstance(editor)
      val count = commandState.commandBuilder.count

      // Reset the command count so it doesn't transfer onto subsequent commands.
      editor.getTopLevelEditor().commandState.commandBuilder.resetCount()

      // Normally we want to jump to the start of the matching pair. But when moving forward in operator
      // pending mode, we want to include the entire match. isInOpPending makes that distinction.
      val isInOpPending = commandState.isOperatorPending

      if (isInOpPending) {
        val matchitAction = MatchitAction()
        matchitAction.reverse = reverse
        matchitAction.isInOpPending = true

        commandState.commandBuilder.completeCommandPart(
          Argument(
            Command(
              count,
              matchitAction, Command.Type.MOTION, EnumSet.noneOf(CommandFlags::class.java)
            )
          )
        )
      } else {
        editor.vimForEachCaret { caret ->
          VimPlugin.getMark().saveJumpLocation(editor)
          MotionGroup.moveCaret(editor, caret, getMatchitOffset(editor, caret, count, isInOpPending, reverse))
        }
      }
    }
  }
}

/**
 * To find a matching pair, we need patterns that describe what the opening and closing pairs look like.
 *
 * We pass around strings instead of compiled Java Patterns since a pattern may require back references to be added
 * before the search can proceed. E.g. for HTML, we use a general pattern to check if the cursor is inside a tag. The
 * pattern captures the tag's name as a back reference so we can later search for something more specific like "</div>"
 */
private typealias MatchitSearchPair = Pair<String, String>

/**
 * A language can have many patterns that describe its matching pairs. We divide the patterns into two maps, one for
 * the opening pairs (e.g. "if" or "def" in Ruby) and one for the closing pairs.
 * For each pair, we take the pattern that describes the text the cursor can be on and map it to the MatchitSearchPair
 * we need to find the match.
 *
 * Simple example: if we wanted the cursor to jump from an opening angle bracket to a closing angle bracket even if the
 * cursor was on whitespace, then we would add "\\s*<" -> MatchitSearchPair("<", ">") to the openingPatterns map.
 */
private data class MatchitPatternsTable(
  val openingPatterns: Map<String, MatchitSearchPair>,
  val closingPatterns: Map<String, MatchitSearchPair>
)

/**
 * All the information we need to find a match.
 */
private data class MatchitSearchParams(
  val caretOffset: Int,
  val endOfCurrentPattern: Int,
  val targetOpeningPattern: String,
  val targetClosingPattern: String,

  // If the cursor is not in a comment, then we want to ignore any matches found in comments.
  // But if we are in comment, then we only want to match on things in comments. The same goes for quotes.
  val skipComments: Boolean,
  val skipStrings: Boolean
)

/**
 * Patterns for the supported file types are stored in this object.
 */
private object MatchitPatterns {

  fun getPatternsForFile(virtualFile: VirtualFile?, reverse: Boolean): MatchitPatternsTable? {
    val fileExtension = virtualFile?.extension
    val fileTypeName = virtualFile?.fileType?.name

    // Ruby file types are only detected if the user is running RubyMine or the Ruby plugin.
    // Checking the extension is a simple fallback that also lets us unit test the Ruby patterns.
    return if (fileTypeName == "Ruby" || fileExtension == "rb") {
      if (reverse) this.reverseRubyPatternsTable else this.rubyPatternsTable
    } else if (fileTypeName == "RHTML" || fileExtension == "erb") {
      if (reverse) this.reverseRubyAndHtmlPatternsTable else this.rubyAndHtmlPatternsTable
    } else if (fileTypeName in htmlLikeFileTypes) {
      this.htmlPatternsTable
    } else {
      return null
    }
  }

  // Files that can contain HTML or HTML-like content.
  // These are just the file types that have HTML Matchit support enabled by default in their Vim ftplugin files.
  private val htmlLikeFileTypes = arrayOf("HTML", "XML", "XHTML", "JSP", "JavaScript", "JSX Harmony", "TypeScript", "TypeScript JSX")

  private val htmlPatternsTable = createHtmlPatternsTable()
  private val rubyPatternsTable = createRubyPatternsTable()
  private val reverseRubyPatternsTable = createRubyPatternsTable(true)
  private val rubyAndHtmlPatternsTable = MatchitPatternsTable(
    rubyPatternsTable.openingPatterns + htmlPatternsTable.openingPatterns,
    rubyPatternsTable.closingPatterns + htmlPatternsTable.closingPatterns
  )
  private val reverseRubyAndHtmlPatternsTable = MatchitPatternsTable(
    reverseRubyPatternsTable.openingPatterns + htmlPatternsTable.openingPatterns,
    reverseRubyPatternsTable.closingPatterns + htmlPatternsTable.closingPatterns
  )
<commit_msg>Add matchit support for handlebars<commit_after>/*
 * IdeaVim - Vim emulator for IDEs based on the IntelliJ platform
 * Copyright (C) 2003-2021 The IdeaVim authors
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

package com.maddyhome.idea.vim.extension.matchit

import com.intellij.openapi.actionSystem.DataContext
import com.intellij.openapi.editor.Caret
import com.intellij.openapi.editor.Editor
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.PsiComment
import com.intellij.psi.PsiElement
import com.intellij.psi.util.PsiTreeUtil
import com.intellij.psi.util.elementType
import com.maddyhome.idea.vim.VimPlugin
import com.maddyhome.idea.vim.command.Argument
import com.maddyhome.idea.vim.command.Command
import com.maddyhome.idea.vim.command.CommandFlags
import com.maddyhome.idea.vim.command.CommandState.Companion.getInstance
import com.maddyhome.idea.vim.command.MappingMode
import com.maddyhome.idea.vim.command.MotionType
import com.maddyhome.idea.vim.command.OperatorArguments
import com.maddyhome.idea.vim.extension.VimExtension
import com.maddyhome.idea.vim.extension.VimExtensionFacade
import com.maddyhome.idea.vim.extension.VimExtensionFacade.putKeyMappingIfMissing
import com.maddyhome.idea.vim.extension.VimExtensionHandler
import com.maddyhome.idea.vim.group.MotionGroup
import com.maddyhome.idea.vim.handler.Motion
import com.maddyhome.idea.vim.handler.MotionActionHandler
import com.maddyhome.idea.vim.handler.toMotionOrError
import com.maddyhome.idea.vim.helper.EditorHelper
import com.maddyhome.idea.vim.helper.PsiHelper
import com.maddyhome.idea.vim.helper.StringHelper.parseKeys
import com.maddyhome.idea.vim.helper.commandState
import com.maddyhome.idea.vim.helper.enumSetOf
import com.maddyhome.idea.vim.helper.getTopLevelEditor
import com.maddyhome.idea.vim.helper.vimForEachCaret
import java.util.*
import java.util.regex.Pattern

/**
 * Port of matchit.vim (https://github.com/chrisbra/matchit)
 * Languages currently supported:
 *   Ruby, Embedded Ruby, XML, and HTML (including HTML in JavaScript/TypeScript, JSX/TSX, and JSPs)
 *
 * @author <NAME> (@myzeiri)
 */
class Matchit : VimExtension {

  override fun getName(): String = "matchit"

  override fun init() {
    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(MatchitMotion)"), owner, MatchitHandler(false), false)
    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(MatchitMotion)"), owner, MatchitHandler(false), false)
    putKeyMappingIfMissing(MappingMode.NXO, parseKeys("%"), owner, parseKeys("<Plug>(MatchitMotion)"), true)

    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(ReverseMatchitMotion)"), owner, MatchitHandler(true), false)
    VimExtensionFacade.putExtensionHandlerMapping(MappingMode.NXO, parseKeys("<Plug>(ReverseMatchitMotion)"), owner, MatchitHandler(true), false)
    putKeyMappingIfMissing(MappingMode.NXO, parseKeys("g%"), owner, parseKeys("<Plug>(ReverseMatchitMotion)"), true)
  }

  private class MatchitAction : MotionActionHandler.ForEachCaret() {
    var reverse = false
    var isInOpPending = false

    override val flags: EnumSet<CommandFlags> = enumSetOf(CommandFlags.FLAG_SAVE_JUMP)

    override fun getOffset(
      editor: Editor,
      caret: Caret,
      context: DataContext,
      argument: Argument?,
      operatorArguments: OperatorArguments,
    ): Motion {
      return getMatchitOffset(editor, caret, operatorArguments.count0, isInOpPending, reverse).toMotionOrError()
    }

    override fun process(cmd: Command) {
      motionType = if (cmd.rawCount != 0) MotionType.LINE_WISE else MotionType.INCLUSIVE
    }

    override var motionType: MotionType = MotionType.INCLUSIVE
  }

  private class MatchitHandler(private val reverse: Boolean) : VimExtensionHandler {

    override fun execute(editor: Editor, context: DataContext) {
      val commandState = getInstance(editor)
      val count = commandState.commandBuilder.count

      // Reset the command count so it doesn't transfer onto subsequent commands.
      editor.getTopLevelEditor().commandState.commandBuilder.resetCount()

      // Normally we want to jump to the start of the matching pair. But when moving forward in operator
      // pending mode, we want to include the entire match. isInOpPending makes that distinction.
      val isInOpPending = commandState.isOperatorPending

      if (isInOpPending) {
        val matchitAction = MatchitAction()
        matchitAction.reverse = reverse
        matchitAction.isInOpPending = true

        commandState.commandBuilder.completeCommandPart(
          Argument(
            Command(
              count,
              matchitAction, Command.Type.MOTION, EnumSet.noneOf(CommandFlags::class.java)
            )
          )
        )
      } else {
        editor.vimForEachCaret { caret ->
          VimPlugin.getMark().saveJumpLocation(editor)
          MotionGroup.moveCaret(editor, caret, getMatchitOffset(editor, caret, count, isInOpPending, reverse))
        }
      }
    }
  }
}

/**
 * To find a matching pair, we need patterns that describe what the opening and closing pairs look like.
 *
 * We pass around strings instead of compiled Java Patterns since a pattern may require back references to be added
 * before the search can proceed. E.g. for HTML, we use a general pattern to check if the cursor is inside a tag. The
 * pattern captures the tag's name as a back reference so we can later search for something more specific like "</div>"
 */
private typealias MatchitSearchPair = Pair<String, String>

/**
 * A language can have many patterns that describe its matching pairs. We divide the patterns into two maps, one for
 * the opening pairs (e.g. "if" or "def" in Ruby) and one for the closing pairs.
 * For each pair, we take the pattern that describes the text the cursor can be on and map it to the MatchitSearchPair
 * we need to find the match.
 *
 * Simple example: if we wanted the cursor to jump from an opening angle bracket to a closing angle bracket even if the
 * cursor was on whitespace, then we would add "\\s*<" -> MatchitSearchPair("<", ">") to the openingPatterns map.
 */
private data class MatchitPatternsTable(
  val openingPatterns: Map<String, MatchitSearchPair>,
  val closingPatterns: Map<String, MatchitSearchPair>
)

/**
 * All the information we need to find a match.
 */
private data class MatchitSearchParams(
  val caretOffset: Int,
  val endOfCurrentPattern: Int,
  val targetOpeningPattern: String,
  val targetClosingPattern: String,

  // If the cursor is not in a comment, then we want to ignore any matches found in comments.
  // But if we are in comment, then we only want to match on things in comments. The same goes for quotes.
  val skipComments: Boolean,
  val skipStrings: Boolean
)

/**
 * Patterns for the supported file types are stored in this object.
 */
private object MatchitPatterns {

  fun getPatternsForFile(virtualFile: VirtualFile?, reverse: Boolean): MatchitPatternsTable? {
    val fileExtension = virtualFile?.extension
    val fileTypeName = virtualFile?.fileType?.name

    // Ruby file types are only detected if the user is running RubyMine or the Ruby plugin.
    // Checking the extension is a simple fallback that also lets us unit test the Ruby patterns.
    return if (fileTypeName == "Ruby" || fileExtension == "rb") {
      if (reverse) this.reverseRubyPatternsTable else this.rubyPatternsTable
    } else if (fileTypeName == "RHTML" || fileExtension == "erb") {
      if (reverse) this.reverseRubyAndHtmlPatternsTable else this.rubyAndHtmlPatternsTable
    } else if (fileTypeName in htmlLikeFileTypes) {
      this.htmlPatternsTable
    } else {
      return null
    }
  }

  // Files that can contain HTML or HTML-like content.
  // These are just the file types that have HTML Matchit support enabled by default in their Vim ftplugin files.
  private val htmlLikeFileTypes = arrayOf("Handlebars/Mustache", "HTML", "XML", "XHTML", "JSP", "JavaScript", "JSX Harmony", "TypeScript", "TypeScript JSX")

  private val htmlPatternsTable = createHtmlPatternsTable()
  private val rubyPatternsTable = createRubyPatternsTable()
  private val reverseRubyPatternsTable = createRubyPatternsTable(true)
  private val rubyAndHtmlPatternsTable = MatchitPatternsTable(
    rubyPatternsTable.openingPatterns + htmlPatternsTable.openingPatterns,
    rubyPatternsTable.closingPatterns + htmlPatternsTable.closingPatterns
  )
  private val reverseRubyAndHtmlPatternsTable = MatchitPatternsTable(
    reverseRubyPatternsTable.openingPatterns + htmlPatternsTable.openingPatterns,
    reverseRubyPatternsTable.closingPatterns + htmlPatternsTable.closingPatterns
  )

{'fraction_non_alphanumeric': 0.08292105829210583, 'fraction_numerical': 0.0011830501183050118, 'mean_word_length': 4.275744680851064, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7249697', 'n_tokens_mistral': 5430, 'n_tokens_neox': 5171, 'n_words': 1633}