<commit_before>Function Get-MatchingFiles
{
    [CmdletBinding()]
    param(
        [string] $Pattern,
        [string] $Root
    )

    [string[]]$matchingFiles = @()
    $Pattern = $Pattern -replace "\r?\n", ";"
    
    if ($Pattern.Contains("*") -or $Pattern.Contains("?"))
    {
        $matchingFiles += @(Find-Files -SearchPattern $Pattern -Root $Root)
        if (!$matchingFiles.Count)
        {
            throw "Target files not found using search pattern '${Pattern}'."
        }
    }
    else
    {
        $matchingFiles += @($Pattern -split ";")
    }
    
    for ($i = 0 ; $i -lt $matchingFiles.Length ; ++$i) {
        if (!$matchingFiles[$i].StartsWith($Root))
        {
            $matchingFiles[$i] = (Join-Path $Root $matchingFiles[$i])
        }
    }

    $matchingFiles
}

Function Get-FileEncoding
{
    [CmdletBinding()]
    param(
        [string] $Path
    )
    
    $bytes = [byte[]](Get-content -Path $Path -Encoding Byte -ReadCount 4 -TotalCount 4)
    if ($bytes[0] -eq 0x2b -and $bytes[1] -eq 0x2f -and $bytes[2] -eq 0x76 -and ($bytes[3] -eq 0x38 -or $bytes[3] -eq 0x39 -or $bytes[3] -eq 0x2b -or $bytes[3] -eq 0x2f))
    {
        return 'UTF7'
    }
    
    if ($Bytes[0] -eq 0xef -and $Bytes[1] -eq 0xbb -and $Bytes[2] -eq 0xbf)
    {
        return 'UTF8'
    }
    
    if ($bytes[0] -eq 0xfe -and $bytes[1] -eq 0xff)
    {
        return 'BigEndianUnicode'
    }
    
    if ($bytes[0] -eq 0xff -and $bytes[1] -eq 0xfe)
    {
        return 'Unicode'
    }
    
    if ($bytes[0] -eq 0 -and $bytes[1] -eq 0 -and $bytes[2] -eq 0xfe -and $bytes[3] -eq 0xff)
    {
        return 'BigEndianUTF32'
    }

    if ($bytes[0] -eq 0xfe -and $bytes[1] -eq 0xff -and $bytes[2] -eq 0 -and $bytes[3] -eq 0)
    {
        return 'UTF32'
    }

    'Ascii'
}

Function Get-Encoding
{
    [CmdletBinding()]
    param(
        [string] $Name,
        [switch] $WriteBOM
    )
    
    switch ($Name)
    {
        "Ascii" { return New-Object System.Text.ASCIIEncoding }
        "UTF7" { return New-Object System.Text.UTF7Encoding($WriteBOM) }
        "UTF8" { return New-Object System.Text.UTF8Encoding($WriteBOM) }
        "Unicode" { return New-Object System.Text.UnicodeEncoding($WriteBOM) }
        "BigEndianUnicode" { return New-Object System.Text.UnicodeEncoding($true, $WriteBOM) }
        "UTF32" { return New-Object System.Text.UTF32Encoding($WriteBOM) }
        "BigEndianUTF32" { return New-Object System.Text.UTF32Encoding($true, $WriteBOM) }
    }
}

Function Set-Variables
{
    [CmdletBinding()]
    param(
        [string] $Path,
        [regex] $Regex,
        [string] $EncodingName,
        [switch] $FailOnMissing,
        [switch] $WriteBOM
    )
    
    Write-Host "Replacing tokens in file '${Path}'..."
    
    $content = [System.IO.File]::ReadAllText($Path)
    $replaceCallback = {
        param(
            [System.Text.RegularExpressions.Match] $Match
        )
        
        $value = Get-TaskVariable $distributedTaskContext $Match.Groups[1].Value
        if (!$value)
        {
            if ($FailOnMissing)
            {
                Write-Error "Variable '$($Match.Groups[1].Value)' not found."
            }
            else
            {
                Write-Warning "Variable '$($Match.Groups[1].Value)' not found."   
            }
        }
        
        Write-Verbose "Replacing '$($Match.Value)' with '${value}'"
        $value
    }
    
    $content = $regex.Replace($content, $replaceCallback)
    <commit_msg>Fix error on empty files<commit_after>Function Get-MatchingFiles
{
    [CmdletBinding()]
    param(
        [string] $Pattern,
        [string] $Root
    )

    [string[]]$matchingFiles = @()
    $Pattern = $Pattern -replace "\r?\n", ";"
    
    if ($Pattern.Contains("*") -or $Pattern.Contains("?"))
    {
        $matchingFiles += @(Find-Files -SearchPattern $Pattern -Root $Root)
        if (!$matchingFiles.Count)
        {
            throw "Target files not found using search pattern '${Pattern}'."
        }
    }
    else
    {
        $matchingFiles += @($Pattern -split ";")
    }
    
    for ($i = 0 ; $i -lt $matchingFiles.Length ; ++$i) {
        if (!$matchingFiles[$i].StartsWith($Root))
        {
            $matchingFiles[$i] = (Join-Path $Root $matchingFiles[$i])
        }
    }

    $matchingFiles
}

Function Get-FileEncoding
{
    [CmdletBinding()]
    param(
        [string] $Path
    )
    
    $bytes = [byte[]](Get-content -Path $Path -Encoding Byte -ReadCount 4 -TotalCount 4)
    if ($bytes[0] -eq 0x2b -and $bytes[1] -eq 0x2f -and $bytes[2] -eq 0x76 -and ($bytes[3] -eq 0x38 -or $bytes[3] -eq 0x39 -or $bytes[3] -eq 0x2b -or $bytes[3] -eq 0x2f))
    {
        return 'UTF7'
    }
    
    if ($Bytes[0] -eq 0xef -and $Bytes[1] -eq 0xbb -and $Bytes[2] -eq 0xbf)
    {
        return 'UTF8'
    }
    
    if ($bytes[0] -eq 0xfe -and $bytes[1] -eq 0xff)
    {
        return 'BigEndianUnicode'
    }
    
    if ($bytes[0] -eq 0xff -and $bytes[1] -eq 0xfe)
    {
        return 'Unicode'
    }
    
    if ($bytes[0] -eq 0 -and $bytes[1] -eq 0 -and $bytes[2] -eq 0xfe -and $bytes[3] -eq 0xff)
    {
        return 'BigEndianUTF32'
    }

    if ($bytes[0] -eq 0xfe -and $bytes[1] -eq 0xff -and $bytes[2] -eq 0 -and $bytes[3] -eq 0)
    {
        return 'UTF32'
    }

    'Ascii'
}

Function Get-Encoding
{
    [CmdletBinding()]
    param(
        [string] $Name,
        [switch] $WriteBOM
    )
    
    switch ($Name)
    {
        "Ascii" { return New-Object System.Text.ASCIIEncoding }
        "UTF7" { return New-Object System.Text.UTF7Encoding($WriteBOM) }
        "UTF8" { return New-Object System.Text.UTF8Encoding($WriteBOM) }
        "Unicode" { return New-Object System.Text.UnicodeEncoding($WriteBOM) }
        "BigEndianUnicode" { return New-Object System.Text.UnicodeEncoding($true, $WriteBOM) }
        "UTF32" { return New-Object System.Text.UTF32Encoding($WriteBOM) }
        "BigEndianUTF32" { return New-Object System.Text.UTF32Encoding($true, $WriteBOM) }
    }
}

Function Set-Variables
{
    [CmdletBinding()]
    param(
        [string] $Path,
        [regex] $Regex,
        [string] $EncodingName,
        [switch] $FailOnMissing,
        [switch] $WriteBOM
    )
    
    Write-Host "Replacing tokens in file '${Path}'..."
    
    $content = [System.IO.File]::ReadAllText($Path)
    if (!$content)
    {
        Write-Verbose "Skipping empty file."
        return
    }
    
    $replaceCallback = {
        param(
            [System.Text.RegularExpressions.Match] $Match
        )
        
        $value = Get-TaskVariable $distributedTaskContext $Match.Groups[1].Value
        if (!$value)
        {
            if ($FailOnMissing)
            {
                Write-Error "Variable '$($Match.Groups[1].Value)' not found."
            }
            else
            {
                Write-Warning "Variable '$($Match.Groups[1].Value)' not found."   
            }
        }
        
        Write-Verbose "Replacing '$($Match.Value)' with '${value}'"
        $value
    }
    
    $content = $regex.Replace($content, $replaceCallback)
    
{'fraction_non_alphanumeric': 0.1531747149091933, 'fraction_numerical': 0.021962551034774038, 'mean_word_length': 1.9319026000825423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2837171', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2436, 'n_words': 537}