<commit_before>package nl.hannahsten.texifyidea.lang.commands

import com.intellij.openapi.project.DumbService
import com.intellij.openapi.project.Project
import com.intellij.psi.search.GlobalSearchScope
import com.intellij.util.indexing.FileBasedIndex
import kotlinx.coroutines.runBlocking
import nl.hannahsten.texifyidea.index.file.LatexExternalCommandIndex
import nl.hannahsten.texifyidea.lang.Dependend
import nl.hannahsten.texifyidea.lang.Described
import nl.hannahsten.texifyidea.lang.LatexPackage
import nl.hannahsten.texifyidea.lang.commands.LatexGenericRegularCommand.*
import nl.hannahsten.texifyidea.psi.LatexCommands
import nl.hannahsten.texifyidea.util.inMathContext
import nl.hannahsten.texifyidea.util.length
import nl.hannahsten.texifyidea.util.startsWithAny
import kotlin.reflect.KClass

/**
 * @author <NAME>, <NAME>
 */
interface LatexCommand : Described, Dependend {

    companion object {

        /**
         * Looks up the given command name in all predefined [LatexCommand]s.
         *
         * @param commandName
         *          The command name to look up. Can start with or without `\`
         * @return The found commands, or `null` when the command doesn't exist.
         */
        fun lookup(commandName: String?): Set<LatexCommand>? {
            var result = commandName ?: return null
            if (result.startsWith("\\")) {
                result = result.substring(1)
            }

            return LatexMathCommand[result] ?: LatexRegularCommand[result]
        }

        /**
         * Create a [LatexCommand] for the given command name, or merge with existing one.
         */
        fun lookupInIndex(cmdWithoutSlash: String, project: Project): Set<LatexCommand> {
            // Don't try to access index when in dumb mode
            if (DumbService.isDumb(project)) return emptySet()

            val cmds = mutableSetOf<LatexCommand>()
            val cmdWithSlash = "\\$cmdWithoutSlash"
            // Look up in index
            FileBasedIndex.getInstance().processValues(LatexExternalCommandIndex.id, cmdWithSlash, null, { file, value ->
                val dependency = LatexPackage.create(file)
                // Merge with already known command if possible, assuming that there was a reason to specify things (especially parameters) manually
                // Basically this means we add the indexed docs to the known command
                val defaultcmds = lookup(cmdWithSlash)?.filter { it.dependency == dependency } ?: emptyList()
                val cmd = if (defaultcmds.isNotEmpty()) {
                    val defaultCommand = defaultcmds.first()
                    object : LatexCommand {
                        override val command = cmdWithoutSlash
                        override val display = defaultCommand.display
                        override val arguments = defaultCommand.arguments
                        override val description = format(value)
                        override val dependency = dependency
                        override val isMathMode = defaultCommand.isMathMode
                    }
                }
                else {
                    object : LatexCommand {
                        override val command = cmdWithoutSlash
                        override val display: String? = null
                        override val arguments = extractArgumentsFromDocs(value, commandWithSlash)
                        override val description = format(value)
                        override val dependency = dependency
                        override val isMathMode = false
                    }
                }
                cmds.add(cmd)
                true
            }, GlobalSearchScope.everythingScope(project))<commit_msg>Make sure to look up hardcoded command if nothing found in index<commit_after>package nl.hannahsten.texifyidea.lang.commands

import com.intellij.openapi.project.DumbService
import com.intellij.openapi.project.Project
import com.intellij.psi.search.GlobalSearchScope
import com.intellij.util.indexing.FileBasedIndex
import kotlinx.coroutines.runBlocking
import nl.hannahsten.texifyidea.index.file.LatexExternalCommandIndex
import nl.hannahsten.texifyidea.lang.Dependend
import nl.hannahsten.texifyidea.lang.Described
import nl.hannahsten.texifyidea.lang.LatexPackage
import nl.hannahsten.texifyidea.lang.commands.LatexGenericRegularCommand.*
import nl.hannahsten.texifyidea.psi.LatexCommands
import nl.hannahsten.texifyidea.util.inMathContext
import nl.hannahsten.texifyidea.util.length
import nl.hannahsten.texifyidea.util.startsWithAny
import kotlin.reflect.KClass

/**
 * @author <NAME>, <NAME>
 */
interface LatexCommand : Described, Dependend {

    companion object {

        /**
         * Looks up the given command name in all predefined [LatexCommand]s.
         *
         * @param commandName
         *          The command name to look up. Can start with or without `\`
         * @return The found commands, or `null` when the command doesn't exist.
         */
        fun lookup(commandName: String?): Set<LatexCommand>? {
            var result = commandName ?: return null
            if (result.startsWith("\\")) {
                result = result.substring(1)
            }

            return LatexMathCommand[result] ?: LatexRegularCommand[result]
        }

        /**
         * Create a [LatexCommand] for the given command name, or merge with existing one.
         */
        fun lookupInIndex(cmdWithoutSlash: String, project: Project): Set<LatexCommand> {
            // Don't try to access index when in dumb mode
            if (DumbService.isDumb(project)) return emptySet()
            val cmdWithSlash = "\\$cmdWithoutSlash"

            // Make sure to look up the hardcoded commands, to have something in case nothing is found in the index
            val cmds = lookup(cmdWithSlash)?.toMutableSet() ?: mutableSetOf()

            // Look up in index
            FileBasedIndex.getInstance().processValues(LatexExternalCommandIndex.id, cmdWithSlash, null, { file, value ->
                val dependency = LatexPackage.create(file)
                // Merge with already known command if possible, assuming that there was a reason to specify things (especially parameters) manually
                // Basically this means we add the indexed docs to the known command
                val defaultcmds = lookup(cmdWithSlash)?.filter { it.dependency == dependency } ?: emptyList()
                val cmd = if (defaultcmds.isNotEmpty()) {
                    val defaultCommand = defaultcmds.first()
                    object : LatexCommand {
                        override val command = cmdWithoutSlash
                        override val display = defaultCommand.display
                        override val arguments = defaultCommand.arguments
                        override val description = format(value)
                        override val dependency = dependency
                        override val isMathMode = defaultCommand.isMathMode
                    }
                }
                else {
                    object : LatexCommand {
                        override val command = cmdWithoutSlash
                        override val display: String? = null
                        override val arguments = extractArgumentsFromDocs(value, commandWithSlash)
                        override val description = format(value)
                        override val dependency = dependency
                        override val isMathMode = false
                    }
                }
                cmds.add(cmd)
                true
            }, GlobalSearchScope.everythingScope(project))
{'fraction_non_alphanumeric': 0.07233766233766234, 'fraction_numerical': 0.00025974025974025974, 'mean_word_length': 2.0438735177865612, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7318577', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1860, 'n_words': 516}