<commit_before>package com.cognifide.gradle.aem.common.file.resolver

import com.cognifide.gradle.aem.common.AemExtension
import com.cognifide.gradle.aem.common.DependencyOptions
import com.cognifide.gradle.aem.common.Formats
import com.cognifide.gradle.aem.common.Patterns
import com.cognifide.gradle.aem.common.file.FileException
import com.cognifide.gradle.aem.common.file.downloader.HttpFileDownloader
import com.cognifide.gradle.aem.common.file.downloader.SftpFileDownloader
import com.cognifide.gradle.aem.common.file.downloader.SmbFileDownloader
import com.cognifide.gradle.aem.common.file.downloader.UrlFileDownloader
import com.cognifide.gradle.aem.common.http.HttpClient
import com.google.common.hash.HashCode
import java.io.File
import org.apache.commons.io.FilenameUtils
import org.apache.commons.lang3.builder.HashCodeBuilder
import org.gradle.api.artifacts.Configuration
import org.gradle.api.tasks.Internal
import org.gradle.util.GFileUtils

/**
 * File downloader with groups supporting files from multiple sources: local and remote (SFTP, SMB, HTTP).
 */
abstract class Resolver<G : FileGroup>(
    @get:Internal
    val aem: AemExtension,

    @get:Internal
    val downloadDir: File
) {
    private val project = aem.project

    private val groupDefault = this.createGroup(GROUP_DEFAULT)

    private var groupCurrent = groupDefault

    private val groupsDefined = mutableListOf<G>().apply { add(groupDefault) }

    val groups: List<G>
        get() = groupsDefined.filter { it.resolutions.isNotEmpty() }

    protected open fun resolve(hash: Any, resolver: (FileResolution) -> File): FileResolution {
        val id = HashCode.fromInt(HashCodeBuilder().append(hash).toHashCode()).toString()

        return groupCurrent.resolve(id, resolver)
    }

    fun outputDirs(filter: G.() -> Boolean = { true }): List<File> {
        return groups.filter(filter).flatMap { it.dirs }
    }

    fun allFiles(filter: G.() -> Boolean = { true }): List<File> {
        return  resolveGroups(filter).flatMap { it.files }
    }

    fun resolveGroups(filter: G.() -> Boolean): List<G> {
        return aem.parallel.pool(PARALLEL_POOL_SIZE, PARALLEL_POOL_NAME, groups.filter(filter)) { it.files; it }
    }

    fun group(name: String): G {
        return groupsDefined.find { it.name == name }
                ?: throw FileException("File group '$name' is not defined.")
    }

    fun dependency(notation: Any): FileResolution {
        return resolve(notation) {
            val configName = "fileResolver_dependency_${HashCodeBuilder().append(notation).append(downloadDir).build()}"
            val configOptions: (Configuration) -> Unit = { it.isTransitive = false }
            val config = project.configurations.create(configName, configOptions)

            project.dependencies.add(config.name, notation)
            config.singleFile<commit_msg>Minor<commit_after>package com.cognifide.gradle.aem.common.file.resolver

import com.cognifide.gradle.aem.common.AemExtension
import com.cognifide.gradle.aem.common.DependencyOptions
import com.cognifide.gradle.aem.common.Formats
import com.cognifide.gradle.aem.common.Patterns
import com.cognifide.gradle.aem.common.file.FileException
import com.cognifide.gradle.aem.common.file.downloader.HttpFileDownloader
import com.cognifide.gradle.aem.common.file.downloader.SftpFileDownloader
import com.cognifide.gradle.aem.common.file.downloader.SmbFileDownloader
import com.cognifide.gradle.aem.common.file.downloader.UrlFileDownloader
import com.cognifide.gradle.aem.common.http.HttpClient
import com.google.common.hash.HashCode
import java.io.File
import org.apache.commons.io.FilenameUtils
import org.apache.commons.lang3.builder.HashCodeBuilder
import org.gradle.api.artifacts.Configuration
import org.gradle.api.tasks.Internal
import org.gradle.util.GFileUtils

/**
 * File downloader with groups supporting files from multiple sources: local and remote (SFTP, SMB, HTTP).
 */
abstract class Resolver<G : FileGroup>(
    @get:Internal
    val aem: AemExtension,

    @get:Internal
    val downloadDir: File
) {
    private val project = aem.project

    private val groupDefault = this.createGroup(GROUP_DEFAULT)

    private var groupCurrent = groupDefault

    private val groupsDefined = mutableListOf<G>().apply { add(groupDefault) }

    val groups: List<G>
        get() = groupsDefined.filter { it.resolutions.isNotEmpty() }

    protected open fun resolve(hash: Any, resolver: (FileResolution) -> File): FileResolution {
        val id = HashCode.fromInt(HashCodeBuilder().append(hash).toHashCode()).toString()

        return groupCurrent.resolve(id, resolver)
    }

    fun outputDirs(filter: G.() -> Boolean = { true }): List<File> {
        return groups.filter(filter).flatMap { it.dirs }
    }

    fun allFiles(filter: G.() -> Boolean = { true }): List<File> {
        return  resolveGroups(filter).flatMap { it.files }
    }

    fun resolveGroups(filter: G.() -> Boolean = { true }): List<G> {
        return aem.parallel.pool(PARALLEL_POOL_SIZE, PARALLEL_POOL_NAME, groups.filter(filter)) { it.files; it }
    }

    fun group(name: String): G {
        return groupsDefined.find { it.name == name }
                ?: throw FileException("File group '$name' is not defined.")
    }

    fun dependency(notation: Any): FileResolution {
        return resolve(notation) {
            val configName = "fileResolver_dependency_${HashCodeBuilder().append(notation).append(downloadDir).build()}"
            val configOptions: (Configuration) -> Unit = { it.isTransitive = false }
            val config = project.configurations.create(configName, configOptions)

            project.dependencies.add(config.name, notation)
            config.singleFile
{'fraction_non_alphanumeric': 0.11289478276042575, 'fraction_numerical': 0.00034897923573547375, 'mean_word_length': 3.796652719665272, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2372424', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1721, 'n_words': 334}