<commit_before>package org.rust.cargo.util

import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.module.Module
import com.intellij.openapi.module.ModuleServiceManager
import com.intellij.openapi.roots.ModuleRootModificationUtil
import com.intellij.openapi.roots.OrderRootType
import com.intellij.openapi.roots.impl.OrderEntryUtil
import com.intellij.openapi.roots.libraries.Library
import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar
import com.intellij.openapi.vfs.JarFileSystem
import com.intellij.openapi.vfs.LocalFileSystem
import com.intellij.openapi.vfs.VirtualFile
import org.rust.cargo.project.CargoProjectDescription

object RustModuleUtil


/**
 * Established Cargo's library name
 */
val Module.cargoLibraryName: String get() = "Cargo <$name>"

/**
 * Established Rust's 'stdlib' library name
 */
private val Module.rustLibraryName: String get() = "Rust <$name>"

/**
 * Helper extracting generic service for the particular module
 */
inline fun<reified T: Any> Module.getService(): T? =
    ModuleServiceManager.getService(this, T::class.java)

inline fun<reified T: Any> Module.getServiceOrThrow(): T =
    getService()!!

/**
 * Helper extracting generic component for the particular module
 */
inline fun<reified T: Any> Module.getComponent(): T? =
    this.getComponent(T::class.java)

inline fun<reified T: Any> Module.getComponentOrThrow(): T =
    getComponent()!!


/**
 * Makes given path relative to the content-root of the module or
 * one of the respective's dependencies
 */
fun Module.relativise(f: VirtualFile): Pair<String, String>? =
    cargoProject?.findPackageForFile(f)?.let {
        val (pkg, relPath) = it
        pkg.name to relPath
    }

/**
 * Rust standard library crates source roots extracted from a zip archive or a folder with rust.
 * Hopefully this class will go away when a standard way to get rust sources appears.
 */
class StandardLibraryRoots private constructor(
    private val roots: List<VirtualFile>
) {

    /**
     * Creates an module level IDEA external library from [roots].
     * The crates can be extracted with [extendProjectDescriptionWithStandardLibraryCrates].
     */
    fun attachTo(module: Module) {
        module.updateLibrary(module.rustLibraryName, roots)
    }

    companion object {
        fun fromPath(path: String): StandardLibraryRoots? =
            LocalFileSystem.getInstance().findFileByPath(path)?.let { fromFile(it) }

        fun fromFile(sources: VirtualFile): StandardLibraryRoots? {
            // sources may be either a zip archive downloaded from github,
            // or a root directory with rust sources, or its src subdirectory.
            // In any case, we want to find the src subdir
            val srcDir = if (sources.isDirectory) {
                if (sources.name == "src") sources else sources.findChild("src")
            } else {
                JarFileSystem.getInstance().getJarRootForLocalFile(sources)
                    ?.children?.singleOrNull()
                    ?.findChild("src")
            } ?: return null

            return StandardLibraryRoots(stdlibCrateNames.map {
                srcDir.findFileByRelativePath("lib$it") ?: return null
            })
        }
    }
}

/**
 * Combines information about project structure which we got form cargo and information
 * about standard library which is stored as an IDEA external library
 */
fun Module.extendProjectDescriptionWithStandardLibraryCrates(projectDescription: CargoProjectDescription) : CargoProjectDescription {
    val lib = LibraryTablesRegistrar.getInstance().getLibraryTable(project).getLibraryByName(rustLibraryName)
        ?: return projectDescription

    val roots: Map<String, VirtualFile> = lib.getFiles(OrderRootType.CLASSES).associateBy { it.name }
    val stdlibPackages = stdlibCrateNames.mapNotNull { name ->
        roots["lib$name"]?.let { libDir ->
            val file = libDir.findFileByRelativePath("lib.rs") ?: return@let null
            name to file
        }
    }

    return projectDescription.withAdditionalPackages(stdlibPackages)
}

/**
 * Updates `CLASS` order-entries for the supplied module's library (referred to with `libraryName`)
 */
fun Module.updateLibrary(libraryName: String, roots: Collection<VirtualFile>) {
    check(ApplicationManager.getApplication().isWriteAccessAllowed)

    val libraryTable = LibraryTablesRegistrar.getInstance().getLibraryTable(project)
    val library = libraryTable.getLibraryByName(libraryName)
        ?: libraryTable.createLibrary(libraryName)
        ?: return

    fillLibrary(library, roots)

    ModuleRootModificationUtil.updateModel(this) { model ->
        OrderEntryUtil.findLibraryOrderEntry(model, library)?.let { previousOrderEntry ->
            model.removeOrderEntry(previousOrderEntry)
        }

        model.addLibraryEntry(library)
    }
}

private val stdlibCrateNames = listOf("std", "core", "collections", "alloc")

private fun fillLibrary(library: Library, roots: Collection<VirtualFile>) {
    val model = library.modifiableModel
    for (url in library.getUrls(OrderRootType.CLASSES)) {
        model.removeRoot(url, OrderRootType.CLASSES)
    }

    for (root in roots) {
        model.addRoot(root, OrderRootType.CLASSES)
    }

    model.commit()
}
<commit_msg>(RES): add librustc_unicode to std crates<commit_after>package org.rust.cargo.util

import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.module.Module
import com.intellij.openapi.module.ModuleServiceManager
import com.intellij.openapi.roots.ModuleRootModificationUtil
import com.intellij.openapi.roots.OrderRootType
import com.intellij.openapi.roots.impl.OrderEntryUtil
import com.intellij.openapi.roots.libraries.Library
import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar
import com.intellij.openapi.vfs.JarFileSystem
import com.intellij.openapi.vfs.LocalFileSystem
import com.intellij.openapi.vfs.VirtualFile
import org.rust.cargo.project.CargoProjectDescription

object RustModuleUtil


/**
 * Established Cargo's library name
 */
val Module.cargoLibraryName: String get() = "Cargo <$name>"

/**
 * Established Rust's 'stdlib' library name
 */
private val Module.rustLibraryName: String get() = "Rust <$name>"

/**
 * Helper extracting generic service for the particular module
 */
inline fun<reified T: Any> Module.getService(): T? =
    ModuleServiceManager.getService(this, T::class.java)

inline fun<reified T: Any> Module.getServiceOrThrow(): T =
    getService()!!

/**
 * Helper extracting generic component for the particular module
 */
inline fun<reified T: Any> Module.getComponent(): T? =
    this.getComponent(T::class.java)

inline fun<reified T: Any> Module.getComponentOrThrow(): T =
    getComponent()!!


/**
 * Makes given path relative to the content-root of the module or
 * one of the respective's dependencies
 */
fun Module.relativise(f: VirtualFile): Pair<String, String>? =
    cargoProject?.findPackageForFile(f)?.let {
        val (pkg, relPath) = it
        pkg.name to relPath
    }

/**
 * Rust standard library crates source roots extracted from a zip archive or a folder with rust.
 * Hopefully this class will go away when a standard way to get rust sources appears.
 */
class StandardLibraryRoots private constructor(
    private val roots: List<VirtualFile>
) {

    /**
     * Creates an module level IDEA external library from [roots].
     * The crates can be extracted with [extendProjectDescriptionWithStandardLibraryCrates].
     */
    fun attachTo(module: Module) {
        module.updateLibrary(module.rustLibraryName, roots)
    }

    companion object {
        fun fromPath(path: String): StandardLibraryRoots? =
            LocalFileSystem.getInstance().findFileByPath(path)?.let { fromFile(it) }

        fun fromFile(sources: VirtualFile): StandardLibraryRoots? {
            // sources may be either a zip archive downloaded from github,
            // or a root directory with rust sources, or its src subdirectory.
            // In any case, we want to find the src subdir
            val srcDir = if (sources.isDirectory) {
                if (sources.name == "src") sources else sources.findChild("src")
            } else {
                JarFileSystem.getInstance().getJarRootForLocalFile(sources)
                    ?.children?.singleOrNull()
                    ?.findChild("src")
            } ?: return null

            return StandardLibraryRoots(stdlibCrateNames.map {
                srcDir.findFileByRelativePath("lib$it") ?: return null
            })
        }
    }
}

/**
 * Combines information about project structure which we got form cargo and information
 * about standard library which is stored as an IDEA external library
 */
fun Module.extendProjectDescriptionWithStandardLibraryCrates(projectDescription: CargoProjectDescription) : CargoProjectDescription {
    val lib = LibraryTablesRegistrar.getInstance().getLibraryTable(project).getLibraryByName(rustLibraryName)
        ?: return projectDescription

    val roots: Map<String, VirtualFile> = lib.getFiles(OrderRootType.CLASSES).associateBy { it.name }
    val stdlibPackages = stdlibCrateNames.mapNotNull { name ->
        roots["lib$name"]?.let { libDir ->
            val file = libDir.findFileByRelativePath("lib.rs") ?: return@let null
            name to file
        }
    }

    return projectDescription.withAdditionalPackages(stdlibPackages)
}

/**
 * Updates `CLASS` order-entries for the supplied module's library (referred to with `libraryName`)
 */
fun Module.updateLibrary(libraryName: String, roots: Collection<VirtualFile>) {
    check(ApplicationManager.getApplication().isWriteAccessAllowed)

    val libraryTable = LibraryTablesRegistrar.getInstance().getLibraryTable(project)
    val library = libraryTable.getLibraryByName(libraryName)
        ?: libraryTable.createLibrary(libraryName)
        ?: return

    fillLibrary(library, roots)

    ModuleRootModificationUtil.updateModel(this) { model ->
        OrderEntryUtil.findLibraryOrderEntry(model, library)?.let { previousOrderEntry ->
            model.removeOrderEntry(previousOrderEntry)
        }

        model.addLibraryEntry(library)
    }
}

private val stdlibCrateNames = listOf("std", "core", "collections", "alloc", "rustc_unicode")

private fun fillLibrary(library: Library, roots: Collection<VirtualFile>) {
    val model = library.modifiableModel
    for (url in library.getUrls(OrderRootType.CLASSES)) {
        model.removeRoot(url, OrderRootType.CLASSES)
    }

    for (root in roots) {
        model.addRoot(root, OrderRootType.CLASSES)
    }

    model.commit()
}

{'fraction_non_alphanumeric': 0.09706186050877687, 'fraction_numerical': 0.0, 'mean_word_length': 3.586310804993543, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4663192', 'n_tokens_mistral': 3013, 'n_tokens_neox': 2901, 'n_words': 738}