<commit_before>/*
 * Use of this source code is governed by the MIT license that can be
 * found in the LICENSE file.
 */

package org.rust.toml.crates.local

import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.json.JsonMapper
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import com.google.common.annotations.VisibleForTesting
import com.intellij.openapi.Disposable
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.components.PersistentStateComponent
import com.intellij.openapi.components.State
import com.intellij.openapi.components.Storage
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.progress.ProgressIndicator
import com.intellij.openapi.progress.Task
import com.intellij.openapi.vfs.VirtualFileManager
import com.intellij.openapi.vfs.newvfs.BulkFileListener
import com.intellij.openapi.vfs.newvfs.events.VFileEvent
import com.intellij.openapi.vfs.newvfs.events.VFilePropertyChangeEvent
import com.intellij.openapiext.isUnitTestMode
import com.intellij.util.io.*
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.api.errors.GitAPIException
import org.eclipse.jgit.lib.ObjectId
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.treewalk.CanonicalTreeParser
import org.eclipse.jgit.treewalk.TreeWalk
import org.eclipse.jgit.treewalk.filter.OrTreeFilter
import org.eclipse.jgit.treewalk.filter.PathFilter
import org.eclipse.jgit.treewalk.filter.TreeFilter
import org.rust.cargo.CargoConstants
import org.rust.openapiext.RsPathManager
import org.rust.stdext.cleanDirectory
import org.rust.toml.crates.local.CratesLocalIndexServiceImpl.Companion.CratesLocalIndexState
import java.io.*
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.atomic.AtomicBoolean

/**
 * Crates local index, created from user cargo registry index on host machine.
 * Used for dependency code insight in project's `Cargo.toml`.
 *
 * Stores crates info in [crates] persistent hash map and hash for commit which has been used for index load in
 * persistent state [CratesLocalIndexState]. Note, state's properties should be mutable in order to be serialized and
 * saved.
 */
@State(name = "CratesLocalIndexState", storages = [Storage("rust.crateslocalindex.xml")])
class CratesLocalIndexServiceImpl
    : CratesLocalIndexService, PersistentStateComponent<CratesLocalIndexState>, Disposable {
    private val userCargoIndexDir: Path
        get() = Paths.get(System.getProperty("user.home"), CARGO_REGISTRY_INDEX_LOCATION)

    // TODO: handle RepositoryNotFoundException
    private val repository: Repository = FileRepositoryBuilder()
        .setGitDir(userCargoIndexDir.toFile())
        .build()

    private val registryHeadCommitHash: String
        get() {
            // BACKCOMPAT: Rust 1.49
            // Since 1.50 there should always be CARGO_REGISTRY_INDEX_TAG
            val objectId = repository.resolve(CARGO_REGISTRY_INDEX_TAG) ?: repository.resolve(CARGO_REGISTRY_INDEX_TAG_PRE_1_50)

            return objectId?.name ?: run {
                LOG.error("Failed to resolve remote branch in the cargo registry index repository")
                INVALID_COMMIT_HASH
            }
        }

    private val crates: PersistentHashMap<String, CargoRegistryCrate>? = run {
        resetIndexIfNeeded()

        val file = baseCratesLocalRegistryDir.resolve("crates-local-index")
        try {
            IOUtil.openCleanOrResetBroken({<commit_msg>Save persistent data of crate local index in other.xml<commit_after>/*
 * Use of this source code is governed by the MIT license that can be
 * found in the LICENSE file.
 */

package org.rust.toml.crates.local

import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.json.JsonMapper
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import com.google.common.annotations.VisibleForTesting
import com.intellij.openapi.Disposable
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.components.*
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.progress.ProgressIndicator
import com.intellij.openapi.progress.Task
import com.intellij.openapi.vfs.VirtualFileManager
import com.intellij.openapi.vfs.newvfs.BulkFileListener
import com.intellij.openapi.vfs.newvfs.events.VFileEvent
import com.intellij.openapi.vfs.newvfs.events.VFilePropertyChangeEvent
import com.intellij.openapiext.isUnitTestMode
import com.intellij.util.io.*
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.api.errors.GitAPIException
import org.eclipse.jgit.lib.ObjectId
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.treewalk.CanonicalTreeParser
import org.eclipse.jgit.treewalk.TreeWalk
import org.eclipse.jgit.treewalk.filter.OrTreeFilter
import org.eclipse.jgit.treewalk.filter.PathFilter
import org.eclipse.jgit.treewalk.filter.TreeFilter
import org.rust.cargo.CargoConstants
import org.rust.openapiext.RsPathManager
import org.rust.stdext.cleanDirectory
import org.rust.toml.crates.local.CratesLocalIndexServiceImpl.Companion.CratesLocalIndexState
import java.io.*
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.atomic.AtomicBoolean

/**
 * Crates local index, created from user cargo registry index on host machine.
 * Used for dependency code insight in project's `Cargo.toml`.
 *
 * Stores crates info in [crates] persistent hash map and hash for commit which has been used for index load in
 * persistent state [CratesLocalIndexState]. Note, state's properties should be mutable in order to be serialized and
 * saved.
 */
@State(name = "CratesLocalIndexState", storages = [
    Storage(StoragePathMacros.NON_ROAMABLE_FILE, roamingType = RoamingType.DISABLED)
])
class CratesLocalIndexServiceImpl
    : CratesLocalIndexService, PersistentStateComponent<CratesLocalIndexState>, Disposable {
    private val userCargoIndexDir: Path
        get() = Paths.get(System.getProperty("user.home"), CARGO_REGISTRY_INDEX_LOCATION)

    // TODO: handle RepositoryNotFoundException
    private val repository: Repository = FileRepositoryBuilder()
        .setGitDir(userCargoIndexDir.toFile())
        .build()

    private val registryHeadCommitHash: String
        get() {
            // BACKCOMPAT: Rust 1.49
            // Since 1.50 there should always be CARGO_REGISTRY_INDEX_TAG
            val objectId = repository.resolve(CARGO_REGISTRY_INDEX_TAG) ?: repository.resolve(CARGO_REGISTRY_INDEX_TAG_PRE_1_50)

            return objectId?.name ?: run {
                LOG.error("Failed to resolve remote branch in the cargo registry index repository")
                INVALID_COMMIT_HASH
            }
        }

    private val crates: PersistentHashMap<String, CargoRegistryCrate>? = run {
        resetIndexIfNeeded()

        val file = baseCratesLocalRegistryDir.resolve("crates-local-index")
        try {
            IOUtil.openCleanOrResetBroken({
{'fraction_non_alphanumeric': 0.08302088912694162, 'fraction_numerical': 0.00241028387787895, 'mean_word_length': 4.512177121771217, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7592886', 'n_tokens_mistral': 2283, 'n_tokens_neox': 2191, 'n_words': 465}