<commit_before>package org.jetbrains.dokka

import org.jetbrains.dokka.LanguageService.RenderMode
import java.util.*

data class FormatLink(val text: String, val href: String)

enum class ListKind {
    Ordered,
    Unordered
}

abstract class StructuredFormatService(locationService: LocationService,
                                       val languageService: LanguageService,
                                       override val extension: String,
                                       val linkExtension: String = extension) : FormatService {
    val locationService: LocationService = locationService.withExtension(linkExtension)

    abstract fun appendBlockCode(to: StringBuilder, line: String, language: String)
    abstract fun appendHeader(to: StringBuilder, text: String, level: Int = 1)
    abstract fun appendParagraph(to: StringBuilder, text: String)
    abstract fun appendLine(to: StringBuilder, text: String = "")
    abstract fun appendAnchor(to: StringBuilder, anchor: String)

    abstract fun appendTable(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableHeader(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableBody(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableRow(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableCell(to: StringBuilder, body: () -> Unit)

    abstract fun formatText(text: String): String
    abstract fun formatSymbol(text: String): String
    abstract fun formatKeyword(text: String): String
    abstract fun formatIdentifier(text: String, kind: IdentifierKind): String
    fun formatEntity(text: String): String = text
    abstract fun formatLink(text: String, href: String): String
    open fun formatLink(link: FormatLink): String = formatLink(formatText(link.text), link.href)
    abstract fun formatStrong(text: String): String
    abstract fun formatStrikethrough(text: String): String
    abstract fun formatEmphasis(text: String): String
    abstract fun formatCode(code: String): String
    abstract fun formatUnorderedList(text: String): String
    abstract fun formatOrderedList(text: String): String
    abstract fun formatListItem(text: String, kind: ListKind): String
    abstract fun formatBreadcrumbs(items: Iterable<FormatLink>): String
    abstract fun formatNonBreakingSpace(): String
    open fun formatSoftLineBreak(): String = ""
    open fun formatIndentedSoftLineBreak(): String = ""

    open fun formatText(location: Location, nodes: Iterable<ContentNode>, listKind: ListKind = ListKind.Unordered): String {
        return nodes.map { formatText(location, it, listKind) }.joinToString("")
    }

    fun formatText(location: Location, content: ContentNode, listKind: ListKind = ListKind.Unordered): String {
        return StringBuilder().apply { formatText(location, content, this, listKind) }.toString()
    }

    open fun formatText(location: Location, content: ContentNode, to: StringBuilder, listKind: ListKind = ListKind.Unordered) {
        when (content) {
            is ContentText -> to.append(formatText(content.text))
            is ContentSymbol -> to.append(formatSymbol(content.text))
            is ContentKeyword -> to.append(formatKeyword(content.text))
            is ContentIdentifier -> to.append(formatIdentifier(content.text, content.kind))
            is ContentNonBreakingSpace -> to.append(formatNonBreakingSpace())
            is ContentSoftLineBreak -> to.append(formatSoftLineBreak())
            is ContentIndentedSoftLineBreak -> to.append(formatIndentedSoftLineBreak())
            is ContentEntity -> to.append(formatEntity(content.text))
            is ContentStrong -> to.append(formatStrong(formatText(location, content.children)))
            is ContentStrikethrough -> to.append(formatStrikethrough(formatText(location, content.children)))
            is ContentCode -> to.append(formatCode(formatText(location, content.children)))
            is ContentEmphasis -> to.append(formatEmphasis(formatText(location, content.children)))
            is ContentUnorderedList -> to.append(formatUnorderedList(formatText(location, content.children, ListKind.Unordered)))
            is ContentOrderedList -> to.append(formatOrderedList(formatText(location, content.children, ListKind.Ordered)))
            is ContentListItem -> to.append(formatListItem(formatText(location, content.children), listKind))

            is ContentNodeLink -> {
                val node = content.node
                val linkTo = if (node != null) locationHref(location, node) else "#"
                val linkText = formatText(location, content.children)
                if (linkTo == ".") {
                    to.append(linkText)
                } else {
                    to.append(formatLink(linkText, linkTo))
                }
            }
            is ContentExternalLink -> {
                val linkText = formatText(location, content.children)
                if (content.href == ".") {
                    to.append(linkText)
                } else {
                    to.append(formatLink(linkText, content.href))
                }
            }
            is ContentParagraph -> appendParagraph(to, formatText(location, content.children))
            is ContentBlockCode -> appendBlockCode(to, formatText(location, content.children), content.language)
            is ContentHeading -> appendHeader(to, formatText(location, content.children), content.level)
            is ContentBlock -> to.append(formatText(location, content.children))
        }
    }

    open fun link(from: DocumentationNode, to: DocumentationNode): FormatLink = link(from, to, extension)

    open fun link(from: DocumentationNode, to: DocumentationNode, extension: String): FormatLink {
        return FormatLink(to.name, locationService.relativePathToLocation(from, to))
    }

    fun locationHref(from: Location, to: DocumentationNode): String {
        val topLevelPage = to.references(RefKind.TopLevelPage).singleOrNull()?.to
        if (topLevelPage != null) {
            return from.relativePathTo(locationService.location(topLevelPage), to.name)
        }
        return from.relativePathTo(locationService.location(to))
    }

    private fun DocumentationNode.isModuleOrPackage(): Boolean =
            kind == NodeKind.Module || kind == NodeKind.Package

    protected open fun appendAsSignature(to: StringBuilder, node: ContentNode, block: () -> Unit) {
        block()
    }

    protected open fun appendAsOverloadGroup(to: StringBuilder, block: () -> Unit) {
        block()
    }

    fun Content.getSectionsWithSubjects(): Map<String, List<ContentSection>> =
            sections.filter { it.subjectName != null }.groupBy { it.tag }

    private fun DocumentationNode.appendOverrides(to: StringBuilder) {
        overrides.forEach {
            to.append("Overrides ")
            val location = locationService.relativePathToLocation(this, it)
            appendLine(to, formatLink(FormatLink(it.owner!!.name + "." + it.name, location)))
        }
    }

    private fun DocumentationNode.appendDeprecation(location: Location, to: StringBuilder) {
        if (deprecation != null) {
            val deprecationParameter = deprecation!!.details(NodeKind.Parameter).firstOrNull()
            val deprecationValue = deprecationParameter?.details(NodeKind.Value)?.firstOrNull()
            if (deprecationValue != null) {
                to.append(formatStrong("Deprecated:")).append(" ")
                appendLine(to, formatText(deprecationValue.name.removeSurrounding("\"")))
                appendLine(to)
            } else if (deprecation?.content != Content.Empty) {
                to.append(formatStrong("Deprecated:")).append(" ")
                formatText(location, deprecation!!.content, to)
            } else {
                appendLine(to, formatStrong("Deprecated"))
                appendLine(to)
            }
        }
    }

    private fun DocumentationNode.appendSourceLink(to: StringBuilder) {
        val sourceUrl = details(NodeKind.SourceUrl).firstOrNull()
        if (sourceUrl != null) {
            to.append(" ")
            appendLine(to, formatLink("(source)", sourceUrl.name))
        } else {
            appendLine(to)
        }
    }

    private fun ContentNode.signatureToText(location: Location): String {
        return if (this is ContentBlock && this.isEmpty()) {
            ""
        } else {
            val signatureAsCode = ContentCode()
            signatureAsCode.append(this)
            formatText(location, signatureAsCode)
        }
    }

    open inner class PageBuilder(val location: Location, val to: StringBuilder, val nodes: Iterable<DocumentationNode>) {
        open fun build() {
            val breakdownByLocation = nodes.groupBy { node ->
                formatBreadcrumbs(node.path.filterNot { it.name.isEmpty() }.map { link(node, it) })
            }

            for ((breadcrumbs, items) in breakdownByLocation) {
                appendLine(to, breadcrumbs)
                appendLine(to)
                appendLocation(items.filter { it.kind != NodeKind.ExternalClass })
            }
        }

        private fun appendLocation(nodes: Iterable<DocumentationNode>) {
            val singleNode = nodes.singleOrNull()
            if (singleNode != null && singleNode.isModuleOrPackage()) {
                if (singleNode.kind == NodeKind.Package) {
                    val packageName = if (singleNode.name.isEmpty()) "<root>" else singleNode.name
                    appendHeader(to, "Package " + formatText(packageName), 2)
                }
                formatText(location, singleNode.content, to)
            } else {
                val breakdownByName = nodes.groupBy { node -> node.name }
                for ((name, items) in breakdownByName) {
                    appendHeader(to, formatText(name))
                    appendDocumentation(items)
                }
            }
        }

        private fun appendDocumentation(overloads: Iterable<DocumentationNode>) {
            val breakdownBySummary = overloads.groupByTo(LinkedHashMap()) { node -> node.content }

            if (breakdownBySummary.size == 1) {
                formatOverloadGroup(breakdownBySummary.values.single())
            } else {
                for ((summary, items) in breakdownBySummary) {
                    appendAsOverloadGroup(to) {
                        formatOverloadGroup(items)
                    }
                }
            }
        }

        private fun formatOverloadGroup(items: MutableList<DocumentationNode>) {
            items.forEach {
                val rendered = languageService.render(it)
                appendAsSignature(to, rendered) {
                    to.append(formatCode(formatText(location, rendered)))
                    it.appendSourceLink(to)
                }
                it.appendOverrides(to)
                it.appendDeprecation(location, to)
            }
            // All items have exactly the same documentation, so we can use any item to render it
            val item = items.first()
            item.details(NodeKind.OverloadGroupNote).forEach {
                formatText(location, it.content, to)
            }
            formatText(location, item.content.summary, to)
            appendDescription(item)
            appendLine(to)
            appendLine(to)
        }

        private fun appendDescription(node: DocumentationNode) {
            if (node.content.description != ContentEmpty) {
                appendLine(to, formatText(location, node.content.description))
                appendLine(to)
            }
            node.content.getSectionsWithSubjects().forEach {
                appendSectionWithSubject(it.key, it.value)
            }

            for (section in node.content.sections.filter { it.subjectName == null }) {
                appendLine(to, formatStrong(formatText(section.tag)))<commit_msg>move some more logic into PageBuilder class<commit_after>package org.jetbrains.dokka

import org.jetbrains.dokka.LanguageService.RenderMode
import java.util.*

data class FormatLink(val text: String, val href: String)

enum class ListKind {
    Ordered,
    Unordered
}

abstract class StructuredFormatService(locationService: LocationService,
                                       val languageService: LanguageService,
                                       override val extension: String,
                                       val linkExtension: String = extension) : FormatService {
    val locationService: LocationService = locationService.withExtension(linkExtension)

    abstract fun appendBlockCode(to: StringBuilder, line: String, language: String)
    abstract fun appendHeader(to: StringBuilder, text: String, level: Int = 1)
    abstract fun appendParagraph(to: StringBuilder, text: String)
    abstract fun appendLine(to: StringBuilder, text: String = "")
    abstract fun appendAnchor(to: StringBuilder, anchor: String)

    abstract fun appendTable(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableHeader(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableBody(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableRow(to: StringBuilder, body: () -> Unit)
    abstract fun appendTableCell(to: StringBuilder, body: () -> Unit)

    abstract fun formatText(text: String): String
    abstract fun formatSymbol(text: String): String
    abstract fun formatKeyword(text: String): String
    abstract fun formatIdentifier(text: String, kind: IdentifierKind): String
    fun formatEntity(text: String): String = text
    abstract fun formatLink(text: String, href: String): String
    open fun formatLink(link: FormatLink): String = formatLink(formatText(link.text), link.href)
    abstract fun formatStrong(text: String): String
    abstract fun formatStrikethrough(text: String): String
    abstract fun formatEmphasis(text: String): String
    abstract fun formatCode(code: String): String
    abstract fun formatUnorderedList(text: String): String
    abstract fun formatOrderedList(text: String): String
    abstract fun formatListItem(text: String, kind: ListKind): String
    abstract fun formatBreadcrumbs(items: Iterable<FormatLink>): String
    abstract fun formatNonBreakingSpace(): String
    open fun formatSoftLineBreak(): String = ""
    open fun formatIndentedSoftLineBreak(): String = ""

    open fun formatText(location: Location, nodes: Iterable<ContentNode>, listKind: ListKind = ListKind.Unordered): String {
        return nodes.map { formatText(location, it, listKind) }.joinToString("")
    }

    fun formatText(location: Location, content: ContentNode, listKind: ListKind = ListKind.Unordered): String {
        return StringBuilder().apply { formatText(location, content, this, listKind) }.toString()
    }

    open fun formatText(location: Location, content: ContentNode, to: StringBuilder, listKind: ListKind = ListKind.Unordered) {
        when (content) {
            is ContentText -> to.append(formatText(content.text))
            is ContentSymbol -> to.append(formatSymbol(content.text))
            is ContentKeyword -> to.append(formatKeyword(content.text))
            is ContentIdentifier -> to.append(formatIdentifier(content.text, content.kind))
            is ContentNonBreakingSpace -> to.append(formatNonBreakingSpace())
            is ContentSoftLineBreak -> to.append(formatSoftLineBreak())
            is ContentIndentedSoftLineBreak -> to.append(formatIndentedSoftLineBreak())
            is ContentEntity -> to.append(formatEntity(content.text))
            is ContentStrong -> to.append(formatStrong(formatText(location, content.children)))
            is ContentStrikethrough -> to.append(formatStrikethrough(formatText(location, content.children)))
            is ContentCode -> to.append(formatCode(formatText(location, content.children)))
            is ContentEmphasis -> to.append(formatEmphasis(formatText(location, content.children)))
            is ContentUnorderedList -> to.append(formatUnorderedList(formatText(location, content.children, ListKind.Unordered)))
            is ContentOrderedList -> to.append(formatOrderedList(formatText(location, content.children, ListKind.Ordered)))
            is ContentListItem -> to.append(formatListItem(formatText(location, content.children), listKind))

            is ContentNodeLink -> {
                val node = content.node
                val linkTo = if (node != null) locationHref(location, node) else "#"
                val linkText = formatText(location, content.children)
                if (linkTo == ".") {
                    to.append(linkText)
                } else {
                    to.append(formatLink(linkText, linkTo))
                }
            }
            is ContentExternalLink -> {
                val linkText = formatText(location, content.children)
                if (content.href == ".") {
                    to.append(linkText)
                } else {
                    to.append(formatLink(linkText, content.href))
                }
            }
            is ContentParagraph -> appendParagraph(to, formatText(location, content.children))
            is ContentBlockCode -> appendBlockCode(to, formatText(location, content.children), content.language)
            is ContentHeading -> appendHeader(to, formatText(location, content.children), content.level)
            is ContentBlock -> to.append(formatText(location, content.children))
        }
    }

    open fun link(from: DocumentationNode, to: DocumentationNode): FormatLink = link(from, to, extension)

    open fun link(from: DocumentationNode, to: DocumentationNode, extension: String): FormatLink {
        return FormatLink(to.name, locationService.relativePathToLocation(from, to))
    }

    fun locationHref(from: Location, to: DocumentationNode): String {
        val topLevelPage = to.references(RefKind.TopLevelPage).singleOrNull()?.to
        if (topLevelPage != null) {
            return from.relativePathTo(locationService.location(topLevelPage), to.name)
        }
        return from.relativePathTo(locationService.location(to))
    }

    private fun DocumentationNode.isModuleOrPackage(): Boolean =
            kind == NodeKind.Module || kind == NodeKind.Package

    protected open fun appendAsSignature(to: StringBuilder, node: ContentNode, block: () -> Unit) {
        block()
    }

    protected open fun appendAsOverloadGroup(to: StringBuilder, block: () -> Unit) {
        block()
    }

    fun Content.getSectionsWithSubjects(): Map<String, List<ContentSection>> =
            sections.filter { it.subjectName != null }.groupBy { it.tag }

    private fun ContentNode.signatureToText(location: Location): String {
        return if (this is ContentBlock && this.isEmpty()) {
            ""
        } else {
            val signatureAsCode = ContentCode()
            signatureAsCode.append(this)
            formatText(location, signatureAsCode)
        }
    }

    open inner class PageBuilder(val location: Location, val to: StringBuilder, val nodes: Iterable<DocumentationNode>) {
        open fun build() {
            val breakdownByLocation = nodes.groupBy { node ->
                formatBreadcrumbs(node.path.filterNot { it.name.isEmpty() }.map { link(node, it) })
            }

            for ((breadcrumbs, items) in breakdownByLocation) {
                appendLine(to, breadcrumbs)
                appendLine(to)
                appendLocation(items.filter { it.kind != NodeKind.ExternalClass })
            }
        }

        private fun appendLocation(nodes: Iterable<DocumentationNode>) {
            val singleNode = nodes.singleOrNull()
            if (singleNode != null && singleNode.isModuleOrPackage()) {
                if (singleNode.kind == NodeKind.Package) {
                    val packageName = if (singleNode.name.isEmpty()) "<root>" else singleNode.name
                    appendHeader(to, "Package " + formatText(packageName), 2)
                }
                formatText(location, singleNode.content, to)
            } else {
                val breakdownByName = nodes.groupBy { node -> node.name }
                for ((name, items) in breakdownByName) {
                    appendHeader(to, formatText(name))
                    appendDocumentation(items)
                }
            }
        }

        private fun appendDocumentation(overloads: Iterable<DocumentationNode>) {
            val breakdownBySummary = overloads.groupByTo(LinkedHashMap()) { node -> node.content }

            if (breakdownBySummary.size == 1) {
                formatOverloadGroup(breakdownBySummary.values.single())
            } else {
                for ((summary, items) in breakdownBySummary) {
                    appendAsOverloadGroup(to) {
                        formatOverloadGroup(items)
                    }
                }
            }
        }

        private fun formatOverloadGroup(items: MutableList<DocumentationNode>) {
            items.forEach {
                val rendered = languageService.render(it)
                appendAsSignature(to, rendered) {
                    to.append(formatCode(formatText(location, rendered)))
                    it.appendSourceLink()
                }
                it.appendOverrides()
                it.appendDeprecation()
            }
            // All items have exactly the same documentation, so we can use any item to render it
            val item = items.first()
            item.details(NodeKind.OverloadGroupNote).forEach {
                formatText(location, it.content, to)
            }
            formatText(location, item.content.summary, to)
            item.appendDescription()
            appendLine(to)
            appendLine(to)
        }

        private fun DocumentationNode.appendSourceLink() {
            val sourceUrl = details(NodeKind.SourceUrl).firstOrNull()
            if (sourceUrl != null) {
                to.append(" ")
                appendLine(to, formatLink("(source)", sourceUrl.name))
            } else {
                appendLine(to)
            }
        }

        private fun DocumentationNode.appendOverrides() {
            overrides.forEach {
                to.append("Overrides ")
                val location = locationService.relativePathToLocation(this, it)
                appendLine(to, formatLink(FormatLink(it.owner!!.name + "." + it.name, location)))
            }
        }

        private fun DocumentationNode.appendDeprecation() {
            if (deprecation != null) {
                val deprecationParameter = deprecation!!.details(NodeKind.Parameter).firstOrNull()
                val deprecationValue = deprecationParameter?.details(NodeKind.Value)?.firstOrNull()
                if (deprecationValue != null) {
                    to.append(formatStrong("Deprecated:")).append(" ")
                    appendLine(to, formatText(deprecationValue.name.removeSurrounding("\"")))
                    appendLine(to)
                } else if (deprecation?.content != Content.Empty) {
                    to.append(formatStrong("Deprecated:")).append(" ")
                    formatText(location, deprecation!!.content, to)
                } else {
                    appendLine(to, formatStrong("Deprecated"))
                    appendLine(to)
                }
            }
        }

        private fun DocumentationNode.appendDescription() {
            if (content.description != ContentEmpty) {
                appendLine(to, formatText(location, content.description))
                appendLine(to)
            }
            content.getSectionsWithSubjects().forEach {
                appendSectionWithSubject(it.key, it.value)
            }

            for (section in content.sections.filter { it.subjectName == null }) {
                appendLine(to, formatStrong(formatText(section.tag)))
{'fraction_non_alphanumeric': 0.09818699493363481, 'fraction_numerical': 0.00025122472051249844, 'mean_word_length': 2.39068710959682, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5040656', 'n_tokens_mistral': 6004, 'n_tokens_neox': 5799, 'n_words': 1354}