<commit_before>package org.jetbrains.dokka.base.translators.psi

import com.intellij.lang.jvm.JvmModifier
import com.intellij.lang.jvm.annotation.JvmAnnotationAttribute
import com.intellij.lang.jvm.types.JvmReferenceType
import com.intellij.openapi.vfs.VirtualFileManager
import com.intellij.psi.*
import com.intellij.psi.impl.source.PsiClassReferenceType
import com.intellij.psi.impl.source.PsiImmediateClassType
import org.jetbrains.dokka.DokkaConfiguration.DokkaSourceSet
import org.jetbrains.dokka.analysis.KotlinAnalysis
import org.jetbrains.dokka.analysis.PsiDocumentableSource
import org.jetbrains.dokka.analysis.from
import org.jetbrains.dokka.links.DRI
import org.jetbrains.dokka.links.nextTarget
import org.jetbrains.dokka.links.withClass
import org.jetbrains.dokka.model.*
import org.jetbrains.dokka.model.doc.DocumentationNode
import org.jetbrains.dokka.model.doc.Param
import org.jetbrains.dokka.model.properties.PropertyContainer
import org.jetbrains.dokka.plugability.DokkaContext
import org.jetbrains.dokka.transformers.sources.SourceToDocumentableTranslator
import org.jetbrains.dokka.utilities.DokkaLogger
import org.jetbrains.kotlin.asJava.elements.KtLightAbstractAnnotation
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.jvm.config.JavaSourceRoot
import org.jetbrains.kotlin.descriptors.Visibilities
import org.jetbrains.kotlin.load.java.JvmAbi
import org.jetbrains.kotlin.load.java.propertyNameByGetMethodName
import org.jetbrains.kotlin.load.java.propertyNamesBySetMethodName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.psi.psiUtil.getChildOfType
import org.jetbrains.kotlin.resolve.DescriptorUtils
import org.jetbrains.kotlin.utils.addToStdlib.safeAs
import java.io.File

class DefaultPsiToDocumentableTranslator(
    private val kotlinAnalysis: KotlinAnalysis
) : SourceToDocumentableTranslator {

    override fun invoke(sourceSet: DokkaSourceSet, context: DokkaContext): DModule {

        fun isFileInSourceRoots(file: File): Boolean =
            sourceSet.sourceRoots.any { root -> file.startsWith(root) }


        val (environment, _) = kotlinAnalysis[sourceSet]

        val sourceRoots = environment.configuration.get(CLIConfigurationKeys.CONTENT_ROOTS)
            ?.filterIsInstance<JavaSourceRoot>()
            ?.mapNotNull { it.file.takeIf(::isFileInSourceRoots) }
            ?: listOf()
        val localFileSystem = VirtualFileManager.getInstance().getFileSystem("file")

        val psiFiles = sourceRoots.map { sourceRoot ->
            sourceRoot.absoluteFile.walkTopDown().mapNotNull {
                localFileSystem.findFileByPath(it.path)?.let { vFile ->
                    PsiManager.getInstance(environment.project).findFile(vFile) as? PsiJavaFile
                }
            }.toList()
        }.flatten()

        val docParser =
            DokkaPsiParser(
                sourceSet,
                context.logger
            )
        return DModule(
            sourceSet.moduleDisplayName,
            psiFiles.mapNotNull { it.safeAs<PsiJavaFile>() }.groupBy { it.packageName }.map { (packageName, psiFiles) ->
                val dri = DRI(packageName = packageName)
                DPackage(
                    dri,
                    emptyList(),
                    emptyList(),
                    psiFiles.flatMap { psiFile ->
                        psiFile.classes.map { docParser.parseClasslike(it, dri) }
                    },
                    emptyList(),
                    emptyMap(),
                    null,
                    setOf(sourceSet)
                )
            },
            emptyMap(),
            null,
            setOf(sourceSet)
        )
    }

    class DokkaPsiParser(
        private val sourceSetData: DokkaSourceSet,
        private val logger: DokkaLogger
    ) {

        private val javadocParser: JavaDocumentationParser = JavadocParser(logger)

        private val cachedBounds = hashMapOf<String, Bound>()

        private fun PsiModifierListOwner.getVisibility() = modifierList?.children?.toList()?.let { ml ->
            when {
                ml.any { it.text == PsiKeyword.PUBLIC } -> JavaVisibility.Public
                ml.any { it.text == PsiKeyword.PROTECTED } -> JavaVisibility.Protected
                ml.any { it.text == PsiKeyword.PRIVATE } -> JavaVisibility.Private
                else -> JavaVisibility.Default
            }
        } ?: JavaVisibility.Default

        private val PsiMethod.hash: Int
            get() = "$returnType $name$parameterList".hashCode()

        private val PsiClassType.shouldBeIgnored: Boolean
            get() = isClass("java.lang.Enum") || isClass("java.lang.Object")

        private fun PsiClassType.isClass(qName: String): Boolean {
            val shortName = qName.substringAfterLast('.')
            if (className == shortName) {
                val psiClass = resolve()
                return psiClass?.qualifiedName == qName
            }
            return false
        }

        private fun <T> T.toSourceSetDependent() = mapOf(sourceSetData to this)

        fun parseClasslike(psi: PsiClass, parent: DRI): DClasslike = with(psi) {
            val dri = parent.withClass(name.toString())
            val ancestryTree = mutableListOf<AncestryLevel>()
            val superMethodsKeys = hashSetOf<Int>()
            val superMethods = mutableListOf<Pair<PsiMethod, DRI>>()
            methods.forEach { superMethodsKeys.add(it.hash) }
            fun parseSupertypes(superTypes: Array<PsiClassType>, level: Int = 0) {
                if (superTypes.isEmpty()) return
                val parsedClasses = superTypes.filter { !it.shouldBeIgnored }.mapNotNull { psi ->
                    psi.resolve()?.let {
                        when {
                            it.isInterface -> DRI.from(it) to JavaClassKindTypes.INTERFACE
                            else -> DRI.from(it) to JavaClassKindTypes.CLASS
                        }?.let {
                            TypeConstructor(
                                it.first,
                                psi.parameters.map(::getProjection)
                            ) to it.second
                        }
                    }
                }
                val (classes, interfaces) = parsedClasses.partition { it.second == JavaClassKindTypes.CLASS }
                ancestryTree.add(AncestryLevel(level, classes.firstOrNull()?.first, interfaces.map { it.first }))
<commit_msg>Apply requested changes<commit_after>package org.jetbrains.dokka.base.translators.psi

import com.intellij.lang.jvm.JvmModifier
import com.intellij.lang.jvm.annotation.JvmAnnotationAttribute
import com.intellij.lang.jvm.types.JvmReferenceType
import com.intellij.openapi.vfs.VirtualFileManager
import com.intellij.psi.*
import com.intellij.psi.impl.source.PsiClassReferenceType
import com.intellij.psi.impl.source.PsiImmediateClassType
import org.jetbrains.dokka.DokkaConfiguration.DokkaSourceSet
import org.jetbrains.dokka.analysis.KotlinAnalysis
import org.jetbrains.dokka.analysis.PsiDocumentableSource
import org.jetbrains.dokka.analysis.from
import org.jetbrains.dokka.links.DRI
import org.jetbrains.dokka.links.nextTarget
import org.jetbrains.dokka.links.withClass
import org.jetbrains.dokka.model.*
import org.jetbrains.dokka.model.doc.DocumentationNode
import org.jetbrains.dokka.model.doc.Param
import org.jetbrains.dokka.model.properties.PropertyContainer
import org.jetbrains.dokka.plugability.DokkaContext
import org.jetbrains.dokka.transformers.sources.SourceToDocumentableTranslator
import org.jetbrains.dokka.utilities.DokkaLogger
import org.jetbrains.kotlin.asJava.elements.KtLightAbstractAnnotation
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.jvm.config.JavaSourceRoot
import org.jetbrains.kotlin.descriptors.Visibilities
import org.jetbrains.kotlin.load.java.JvmAbi
import org.jetbrains.kotlin.load.java.propertyNameByGetMethodName
import org.jetbrains.kotlin.load.java.propertyNamesBySetMethodName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.psi.psiUtil.getChildOfType
import org.jetbrains.kotlin.resolve.DescriptorUtils
import org.jetbrains.kotlin.utils.addToStdlib.safeAs
import java.io.File

class DefaultPsiToDocumentableTranslator(
    private val kotlinAnalysis: KotlinAnalysis
) : SourceToDocumentableTranslator {

    override fun invoke(sourceSet: DokkaSourceSet, context: DokkaContext): DModule {

        fun isFileInSourceRoots(file: File): Boolean =
            sourceSet.sourceRoots.any { root -> file.startsWith(root) }


        val (environment, _) = kotlinAnalysis[sourceSet]

        val sourceRoots = environment.configuration.get(CLIConfigurationKeys.CONTENT_ROOTS)
            ?.filterIsInstance<JavaSourceRoot>()
            ?.mapNotNull { it.file.takeIf(::isFileInSourceRoots) }
            ?: listOf()
        val localFileSystem = VirtualFileManager.getInstance().getFileSystem("file")

        val psiFiles = sourceRoots.map { sourceRoot ->
            sourceRoot.absoluteFile.walkTopDown().mapNotNull {
                localFileSystem.findFileByPath(it.path)?.let { vFile ->
                    PsiManager.getInstance(environment.project).findFile(vFile) as? PsiJavaFile
                }
            }.toList()
        }.flatten()

        val docParser =
            DokkaPsiParser(
                sourceSet,
                context.logger
            )
        return DModule(
            sourceSet.moduleDisplayName,
            psiFiles.mapNotNull { it.safeAs<PsiJavaFile>() }.groupBy { it.packageName }.map { (packageName, psiFiles) ->
                val dri = DRI(packageName = packageName)
                DPackage(
                    dri,
                    emptyList(),
                    emptyList(),
                    psiFiles.flatMap { psiFile ->
                        psiFile.classes.map { docParser.parseClasslike(it, dri) }
                    },
                    emptyList(),
                    emptyMap(),
                    null,
                    setOf(sourceSet)
                )
            },
            emptyMap(),
            null,
            setOf(sourceSet)
        )
    }

    class DokkaPsiParser(
        private val sourceSetData: DokkaSourceSet,
        private val logger: DokkaLogger
    ) {

        private val javadocParser: JavaDocumentationParser = JavadocParser(logger)

        private val cachedBounds = hashMapOf<String, Bound>()

        private fun PsiModifierListOwner.getVisibility() = modifierList?.children?.toList()?.let { ml ->
            when {
                ml.any { it.text == PsiKeyword.PUBLIC } -> JavaVisibility.Public
                ml.any { it.text == PsiKeyword.PROTECTED } -> JavaVisibility.Protected
                ml.any { it.text == PsiKeyword.PRIVATE } -> JavaVisibility.Private
                else -> JavaVisibility.Default
            }
        } ?: JavaVisibility.Default

        private val PsiMethod.hash: Int
            get() = "$returnType $name$parameterList".hashCode()

        private val PsiClassType.shouldBeIgnored: Boolean
            get() = isClass("java.lang.Enum") || isClass("java.lang.Object")

        private fun PsiClassType.isClass(qName: String): Boolean {
            val shortName = qName.substringAfterLast('.')
            if (className == shortName) {
                val psiClass = resolve()
                return psiClass?.qualifiedName == qName
            }
            return false
        }

        private fun <T> T.toSourceSetDependent() = mapOf(sourceSetData to this)

        fun parseClasslike(psi: PsiClass, parent: DRI): DClasslike = with(psi) {
            val dri = parent.withClass(name.toString())
            val ancestryTree = mutableListOf<AncestryLevel>()
            val superMethodsKeys = hashSetOf<Int>()
            val superMethods = mutableListOf<Pair<PsiMethod, DRI>>()
            methods.forEach { superMethodsKeys.add(it.hash) }
            fun parseSupertypes(superTypes: Array<PsiClassType>, level: Int = 0) {
                if (superTypes.isEmpty()) return
                val parsedClasses = superTypes.filter { !it.shouldBeIgnored }.mapNotNull { psi ->
                    psi.resolve()?.let { psiClass ->
                        val (dri, javaClassKind) = when {
                            psiClass.isInterface -> DRI.from(psiClass) to JavaClassKindTypes.INTERFACE
                            else -> DRI.from(psiClass) to JavaClassKindTypes.CLASS
                        }
                        TypeConstructor(
                            dri,
                            psi.parameters.map(::getProjection)
                        ) to javaClassKind
                    }
                }
                val (classes, interfaces) = parsedClasses.partition { it.second == JavaClassKindTypes.CLASS }
                ancestryTree.add(AncestryLevel(level, classes.firstOrNull()?.first, interfaces.map { it.first }))

{'fraction_non_alphanumeric': 0.09256834809395456, 'fraction_numerical': 0.00015402387370042357, 'mean_word_length': 2.280121242738065, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '20837', 'n_tokens_mistral': 3754, 'n_tokens_neox': 3544, 'n_words': 591}