<commit_before>package nl.hannahsten.texifyidea.inspections.latex

import com.intellij.codeInsight.daemon.quickFix.CreateFilePathFix
import com.intellij.codeInsight.daemon.quickFix.NewFileLocation
import com.intellij.codeInsight.daemon.quickFix.TargetDirectory
import com.intellij.codeInspection.InspectionManager
import com.intellij.codeInspection.LocalQuickFix
import com.intellij.codeInspection.ProblemDescriptor
import com.intellij.codeInspection.ProblemHighlightType
import com.intellij.openapi.project.Project
import com.intellij.openapi.roots.ProjectRootManager
import com.intellij.openapi.util.TextRange
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.PsiDocumentManager
import com.intellij.psi.PsiFile
import nl.hannahsten.texifyidea.insight.InsightGroup
import nl.hannahsten.texifyidea.inspections.TexifyInspectionBase
import nl.hannahsten.texifyidea.lang.LatexCommand
import nl.hannahsten.texifyidea.lang.RequiredArgument
import nl.hannahsten.texifyidea.lang.RequiredFileArgument
import nl.hannahsten.texifyidea.lang.magic.MagicCommentScope
import nl.hannahsten.texifyidea.psi.LatexCommands
import nl.hannahsten.texifyidea.psi.LatexNormalText
import nl.hannahsten.texifyidea.psi.LatexParameter
import nl.hannahsten.texifyidea.psi.impl.LatexCommandsImpl
import nl.hannahsten.texifyidea.ui.CreateFileDialog
import nl.hannahsten.texifyidea.util.*
import nl.hannahsten.texifyidea.util.Magic.Command.illegalExtensions
import nl.hannahsten.texifyidea.util.Magic.Command.includeOnlyExtensions
import nl.hannahsten.texifyidea.util.files.*
import java.io.File
import java.util.*
import kotlin.collections.ArrayList

/**
 * @author <NAME>
 */
open class LatexFileNotFoundInspection : TexifyInspectionBase() {

    override val inspectionGroup = InsightGroup.LATEX

    override val inspectionId = "FileNotFound"

    override val outerSuppressionScopes = EnumSet.of(MagicCommentScope.GROUP)!!

    override fun getDisplayName() = "File not found"

    override fun inspectFile(file: PsiFile, manager: InspectionManager, isOntheFly: Boolean): MutableList<ProblemDescriptor> {
        val descriptors = descriptorList()

        // Get all comands of project.
        val allCommands = file.commandsInFileSet()
        // Get commands of this file.
        val commands = file.commandsInFile()

        val graphPaths: ArrayList<String> = ArrayList()

        // Check if a graphicspath is defined
        val collection = allCommands.filter { it.name == "\\graphicspath" }

        // Is a graphicspath defined?
        if (collection.isNotEmpty()) {
            // Check if there is even an includegraphics in local commandset
            if (commands.any { it.name == "\\includegraphics" }) {
                val args = collection.last().parameterList.filter { it.requiredParam != null }
                val subArgs = args.first().childrenOfType(LatexNormalText::class)
                subArgs.forEach { graphPaths.add(it.text) }
            }
        }

        // Loop through commands of file
        for (command in commands) {
            // Only consider default commands with a file argument.
            val default = LatexCommand.lookup(command.name) ?: continue

            // Remove optional arguments from list of commands
            val arguments = default.arguments.mapNotNull { it as? RequiredArgument }

            // Remove optional parameters from list of parameters
            val parameters = command.parameterList.filter { it.requiredParam != null }

            // Loop through arguments
            for (i in arguments.indices) {
                // when there are more required arguments than actual present break the loop
                if (i >= parameters.size) {
                    break
                }

                // check if actual argument is a file argument or continue with next argument
                val fileArgument = arguments[i] as? RequiredFileArgument ?: continue
                val extensions = fileArgument.supportedExtensions
                val parameter = parameters[i]

                goThroughFileNames(file, parameter, command, graphPaths, extensions, manager, isOntheFly, descriptors)
            }
        }

        return descriptors
    }

    private fun goThroughFileNames(file: PsiFile, parameter: LatexParameter, command: LatexCommands, graphPaths: ArrayList<String>,
                                   extensions: Set<String>, manager: InspectionManager, isOntheFly: Boolean, descriptors: MutableList<ProblemDescriptor>) {
        // get file name of the command or continue with next parameter
        val fileNames = parameter.splitContent()

        // get root file of the document actual worked with
        val root = file.findRootFile()

        // get the virtual file of the root file
        val containingDirectory = root.containingDirectory.virtualFile

        for (fileName in fileNames) {
            // Check if command is a includegraphics - next file if it exists
            if (command.name.equals("\\includegraphics")) {
                if (findGraphicsFile(containingDirectory, graphPaths, fileName)) continue
            }
            else {
                if (findGeneralFile(containingDirectory, file, extensions, fileName)) continue
            }

            val newFileLocation = NewFileLocation(listOf(TargetDirectory(root.containingDirectory)), fileName)
            val fixes = mutableListOf(CreateFilePathFix(file, newFileLocation))

            // Create quick fixes for all extensions if none was supplied in the argument
            if (extensions.none { fileName.endsWith(".$it") }) {
                extensions.forEach {
                    val fileLocation = NewFileLocation(listOf(TargetDirectory(root.containingDirectory)), "$fileName.$it")
                    fixes.add(CreateFilePathFix(file, fileLocation))
                }
            }

            // Find extension
            val extension = if (command.commandToken.text in includeOnlyExtensions.keys) {
                includeOnlyExtensions[command.commandToken.text]?.toList()?.first() ?: "tex"
            }
            else fileName.getFileExtention()

            val parameterOffset = parameter.text.trimRange(1, 1).indexOf(fileName)

            descriptors.add(manager.createProblemDescriptor(
                    parameter,
                    TextRange(parameterOffset + 1, parameterOffset + fileName.length + 1),
                    "File '${fileName.appendExtension(extension)}' not found",
                    ProblemHighlightType.GENERIC_ERROR,
                    isOntheFly,
                    InspectionFix(fileName, extension)
            ))
        }
    }

    private fun findGeneralFile(containingDir: VirtualFile, file: PsiFile, validExtenions: Set<String>, fileName: String): Boolean {
        val jfile = File(fileName)
        if (jfile.isAbsolute) {
            if (jfile.exists()) return true
        }
        else {
            // check if the given name is reachable from the given folder
            var relative = containingDir.findFile(fileName, validExtenions)

            // If not, check if it is reachable from any content root which will be included when using MiKTeX
            if (LatexDistribution.isMiktex) {
                for (moduleRoot in ProjectRootManager.getInstance(file.project).contentSourceRoots) {
                    if (relative != null) {
                        break
                    }
                    relative = moduleRoot.findFile(fileName, validExtenions)
                }
            }

            // If file was found continue with next file
            return (relative != null)
        }
        return false
    }

    private fun findGraphicsFile(containingDir: VirtualFile, searchPaths: ArrayList<String>, fileName: String): Boolean {
        val file = File(fileName)
        if (file.isAbsolute) {
            // If file was found continue with next file
            if (file.exists()) return true
        }
        else {
            searchPaths.forEach {
                // check if the given name is reachable from the given folder
                if (containingDir.findFileByRelativePath(it + fileName) != null) return true
            }
        }

        // check also the root folder
        return (containingDir.findFileByRelativePath(fileName) != null)
    }

    /**
     * Create a new file.
     */
    class InspectionFix(private val filePath: String, private val extension: String) : LocalQuickFix {

        override fun getFamilyName() = "Create file ${filePath.appendExtension(extension)}"

        override fun startInWriteAction() = false

        override fun applyFix(project: Project, descriptor: ProblemDescriptor) {
            val cmd = descriptor.psiElement as LatexParameter
            val file = cmd.containingFile
            val root = file.findRootFile().containingDirectory.virtualFile.canonicalPath ?: return
            val document = PsiDocumentManager.getInstance(project).getDocument(file) ?: return

            // Display a dialog to ask for the location and name of the new file.
            val newFilePath = CreateFileDialog(file.containingDirectory.virtualFile.canonicalPath, filePath.formatAsFilePath())
                    .newFileFullPath ?: return

            val parameterIndex = cmd.text.indexOf(filePath)<commit_msg>include search of files with omitted extension<commit_after>package nl.hannahsten.texifyidea.inspections.latex

import com.intellij.codeInsight.daemon.quickFix.CreateFilePathFix
import com.intellij.codeInsight.daemon.quickFix.NewFileLocation
import com.intellij.codeInsight.daemon.quickFix.TargetDirectory
import com.intellij.codeInspection.InspectionManager
import com.intellij.codeInspection.LocalQuickFix
import com.intellij.codeInspection.ProblemDescriptor
import com.intellij.codeInspection.ProblemHighlightType
import com.intellij.openapi.project.Project
import com.intellij.openapi.roots.ProjectRootManager
import com.intellij.openapi.util.TextRange
import com.intellij.openapi.vfs.LocalFileSystem
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.PsiDocumentManager
import com.intellij.psi.PsiFile
import nl.hannahsten.texifyidea.insight.InsightGroup
import nl.hannahsten.texifyidea.inspections.TexifyInspectionBase
import nl.hannahsten.texifyidea.lang.LatexCommand
import nl.hannahsten.texifyidea.lang.RequiredArgument
import nl.hannahsten.texifyidea.lang.RequiredFileArgument
import nl.hannahsten.texifyidea.lang.magic.MagicCommentScope
import nl.hannahsten.texifyidea.psi.LatexCommands
import nl.hannahsten.texifyidea.psi.LatexNormalText
import nl.hannahsten.texifyidea.psi.LatexParameter
import nl.hannahsten.texifyidea.psi.impl.LatexCommandsImpl
import nl.hannahsten.texifyidea.ui.CreateFileDialog
import nl.hannahsten.texifyidea.util.*
import nl.hannahsten.texifyidea.util.Magic.Command.illegalExtensions
import nl.hannahsten.texifyidea.util.Magic.Command.includeOnlyExtensions
import nl.hannahsten.texifyidea.util.files.*
import java.io.File
import java.util.*
import kotlin.collections.ArrayList

/**
 * @author <NAME>
 */
open class LatexFileNotFoundInspection : TexifyInspectionBase() {

    override val inspectionGroup = InsightGroup.LATEX

    override val inspectionId = "FileNotFound"

    override val outerSuppressionScopes = EnumSet.of(MagicCommentScope.GROUP)!!

    override fun getDisplayName() = "File not found"

    override fun inspectFile(file: PsiFile, manager: InspectionManager, isOntheFly: Boolean): MutableList<ProblemDescriptor> {
        val descriptors = descriptorList()

        // Get all comands of project.
        val allCommands = file.commandsInFileSet()
        // Get commands of this file.
        val commands = file.commandsInFile()

        val graphPaths: ArrayList<String> = ArrayList()

        // Check if a graphicspath is defined
        val collection = allCommands.filter { it.name == "\\graphicspath" }

        // Is a graphicspath defined?
        if (collection.isNotEmpty()) {
            // Check if there is even an includegraphics in local commandset
            if (commands.any { it.name == "\\includegraphics" }) {
                val args = collection.last().parameterList.filter { it.requiredParam != null }
                val subArgs = args.first().childrenOfType(LatexNormalText::class)
                subArgs.forEach { graphPaths.add(it.text) }
            }
        }

        // Loop through commands of file
        for (command in commands) {
            // Only consider default commands with a file argument.
            val default = LatexCommand.lookup(command.name) ?: continue

            // Remove optional arguments from list of commands
            val arguments = default.arguments.mapNotNull { it as? RequiredArgument }

            // Remove optional parameters from list of parameters
            val parameters = command.parameterList.filter { it.requiredParam != null }

            // Loop through arguments
            for (i in arguments.indices) {
                // when there are more required arguments than actual present break the loop
                if (i >= parameters.size) {
                    break
                }

                // check if actual argument is a file argument or continue with next argument
                val fileArgument = arguments[i] as? RequiredFileArgument ?: continue
                val extensions = fileArgument.supportedExtensions
                val parameter = parameters[i]

                goThroughFileNames(file, parameter, command, graphPaths, extensions, manager, isOntheFly, descriptors)
            }
        }

        return descriptors
    }

    private fun goThroughFileNames(file: PsiFile, parameter: LatexParameter, command: LatexCommands, graphPaths: ArrayList<String>,
                                   extensions: Set<String>, manager: InspectionManager, isOntheFly: Boolean, descriptors: MutableList<ProblemDescriptor>) {
        // get file name of the command or continue with next parameter
        val fileNames = parameter.splitContent()

        // get root file of the document actual worked with
        val root = file.findRootFile()

        // get the virtual file of the root file
        val containingDirectory = root.containingDirectory.virtualFile

        for (fileName in fileNames) {
            // Check if command is a includegraphics - next file if it exists
            if (command.name.equals("\\includegraphics")) {
                if (findGraphicsFile(containingDirectory, extensions, graphPaths, fileName)) continue
            }
            else {
                if (findGeneralFile(containingDirectory, file, extensions, fileName)) continue
            }

            val newFileLocation = NewFileLocation(listOf(TargetDirectory(root.containingDirectory)), fileName)
            val fixes = mutableListOf(CreateFilePathFix(file, newFileLocation))

            // Create quick fixes for all extensions if none was supplied in the argument
            if (extensions.none { fileName.endsWith(".$it") }) {
                extensions.forEach {
                    val fileLocation = NewFileLocation(listOf(TargetDirectory(root.containingDirectory)), "$fileName.$it")
                    fixes.add(CreateFilePathFix(file, fileLocation))
                }
            }

            // Find extension
            val extension = if (command.commandToken.text in includeOnlyExtensions.keys) {
                includeOnlyExtensions[command.commandToken.text]?.toList()?.first() ?: "tex"
            }
            else fileName.getFileExtention()

            val parameterOffset = parameter.text.trimRange(1, 1).indexOf(fileName)

            descriptors.add(manager.createProblemDescriptor(
                    parameter,
                    TextRange(parameterOffset + 1, parameterOffset + fileName.length + 1),
                    "File '${fileName.appendExtension(extension)}' not found",
                    ProblemHighlightType.GENERIC_ERROR,
                    isOntheFly,
                    InspectionFix(fileName, extension)
            ))
        }
    }

    private fun findGeneralFile(containingDir: VirtualFile, file: PsiFile, validExtenions: Set<String>, fileName: String): Boolean {
        val jfile = File(fileName)
        if (jfile.isAbsolute) {
            val fs = LocalFileSystem.getInstance()

            if (fs.findFileByPath(fileName) != null) return true
            validExtenions.forEach {
                if (fs.findFileByPath("$fileName.$it") != null) return true
            }
        }
        else {
            // check if the given name is reachable from the given folder
            var relative = containingDir.findFile(fileName, validExtenions)

            // If not, check if it is reachable from any content root which will be included when using MiKTeX
            if (LatexDistribution.isMiktex) {
                for (moduleRoot in ProjectRootManager.getInstance(file.project).contentSourceRoots) {
                    if (relative != null) {
                        break
                    }
                    relative = moduleRoot.findFile(fileName, validExtenions)
                }
            }

            // If file was found continue with next file
            return (relative != null)
        }
        return false
    }

    private fun findGraphicsFile(containingDir: VirtualFile, validExtenions: Set<String>, searchPaths: ArrayList<String>, fileName: String): Boolean {
        val file = File(fileName)
        if (file.isAbsolute) {
            val fs = LocalFileSystem.getInstance()

            // If file was found continue with next file
            if (fs.findFileByPath(fileName) != null) return true
            validExtenions.forEach {
                if (fs.findFileByPath("$fileName.$it") != null) return true
            }
        }
        else {
            searchPaths.forEach {
                // check if the given name is reachable from the given folder
                if (containingDir.findFile(it + fileName, validExtenions) != null) return true
            }

            // check also the root folder
            return (containingDir.findFile(fileName, validExtenions) != null)
        }
        return false
    }

    /**
     * Create a new file.
     */
    class InspectionFix(private val filePath: String, private val extension: String) : LocalQuickFix {

        override fun getFamilyName() = "Create file ${filePath.appendExtension(extension)}"

        override fun startInWriteAction() = false

        override fun applyFix(project: Project, descriptor: ProblemDescriptor) {
            val cmd = descriptor.psiElement as LatexParameter
            val file = cmd.containingFile
            val root = file.findRootFile().containingDirectory.virtualFile.canonicalPath ?: return
            val document = PsiDocumentManager.getInstance(project).getDocument(file) ?: return

            // Display a dialog to ask for the location and name of the new file.
            val newFilePath = CreateFileDialog(file.containingDirectory.virtualFile.canonicalPath, filePath.formatAsFilePath())
                    .newFileFullPath ?: return

            val parameterIndex = cmd.text.indexOf(filePath)
{'fraction_non_alphanumeric': 0.07887631114342092, 'fraction_numerical': 0.0004154117769238758, 'mean_word_length': 2.477609245214879, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '19809', 'n_tokens_mistral': 4965, 'n_tokens_neox': 4768, 'n_words': 1260}