<commit_before>## Powershell Package Initialize

Function Find-ExecutablePath {
    param(
        [String]$Path
    )
    if (!(Test-Path $Path)) {
        return $null
    }
    $files = Get-ChildItem -Path "$Path\*.exe"
    if ($files.Count -ge 1) {
        return $Path
    }
    if ((Test-Path "$Path\bin")) {
        return "$Path\bin"
    }
    if ((Test-Path "$Path\cmd")) {
        return "$Path\cmd"
    }
    return $null
}


Function Test-AddPath {
    param(
        [String]$Path
    )
    if (Test-Path $Path) {
        $env:Path = "$Path;${env:Path}"
    }
}

Function Test-ExecuteFile {
    param(
        [Parameter(Position = 0, Mandatory = $True, HelpMessage = "Enter Execute Name")]
        [ValidateNotNullorEmpty()]
        [String]$ExeName
    )
    $myErr = @()
    Get-command -CommandType Application $ExeName -ErrorAction SilentlyContinue -ErrorVariable +myErr
    if ($myErr.count -eq 0) {
        return $True
    }
    return $False
}

Function Get-RegistryValueEx {
    param(
        [ValidateNotNullorEmpty()]
        [String]$Path,
        [ValidateNotNullorEmpty()]
        [String]$Key
    )
    if (!(Test-Path $Path)) {
        return 
    }
    (Get-ItemProperty $Path $Key).$Key
}


Function InitializePackageEnv {
    param(
        [String]$ClangbuilderRoot
    )
    $obj = Get-Content -Path "$ClangbuilderRoot\pkgs\packages.lock.json" |ConvertFrom-Json
    Get-Member -InputObject $obj -MemberType NoteProperty|ForEach-Object {
        $xpath = Find-ExecutablePath -Path "$ClangbuilderRoot\pkgs\$($_.Name)"
        if ($null -ne $xpath) {
            Test-AddPath -Path $xpath
        }
    }
    if (!(Test-ExecuteFile "git")) {
        $gitkey = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\Git_is1"
        $gitkey2 = "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\Git_is1"
        if (Test-Path $gitkey) {
            $gitinstall = Get-RegistryValueEx $gitkey "InstallLocation"
            Test-AddPath "${gitinstall}\bin"
        }
        elseif (Test-Path $gitkey2) {
            $gitinstall = Get-RegistryValueEx $gitkey2 "InstallLocation"
            Test-AddPath "${gitinstall}\bin"
        }
    }
}


Function PMDownload {
    param(
        [String]$Uri, ### URI
        [String]$Path ### save to path
    )
    Write-Host "Download $Uri ..."
    $InternalUA= [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome
    if($Uri.Contains("sourceforge.net")){
        $InternalUA="Clangbuilder/5.0"
    }
    Write-Host "UserAgent: $InternalUA"
    try {
        Invoke-WebRequest -Uri $Uri -OutFile $Path -UserAgent $InternalUA -UseBasicParsing
    }
    catch {
        Write-Host -ForegroundColor Red "Download error: $_"
        return $false
    }
    return $true
}

Function Expand-Msi {
    param(
        [String]$Path,
        [String]$DestinationPath ### Full dir of destination path
    )
    $process = Start-Process -FilePath "msiexec" -ArgumentList "/a `"$Path`" /qn TARGETDIR=`"$DestinationPath`""  -PassThru -Wait
    if ($process.ExitCode -ne 0) {
        Write-Host -ForegroundColor Red "Expand-Msi: $Path failed. $($process.ExitCode)"<commit_msg>remove ua debug<commit_after>## Powershell Package Initialize

Function Find-ExecutablePath {
    param(
        [String]$Path
    )
    if (!(Test-Path $Path)) {
        return $null
    }
    $files = Get-ChildItem -Path "$Path\*.exe"
    if ($files.Count -ge 1) {
        return $Path
    }
    if ((Test-Path "$Path\bin")) {
        return "$Path\bin"
    }
    if ((Test-Path "$Path\cmd")) {
        return "$Path\cmd"
    }
    return $null
}


Function Test-AddPath {
    param(
        [String]$Path
    )
    if (Test-Path $Path) {
        $env:Path = "$Path;${env:Path}"
    }
}

Function Test-ExecuteFile {
    param(
        [Parameter(Position = 0, Mandatory = $True, HelpMessage = "Enter Execute Name")]
        [ValidateNotNullorEmpty()]
        [String]$ExeName
    )
    $myErr = @()
    Get-command -CommandType Application $ExeName -ErrorAction SilentlyContinue -ErrorVariable +myErr
    if ($myErr.count -eq 0) {
        return $True
    }
    return $False
}

Function Get-RegistryValueEx {
    param(
        [ValidateNotNullorEmpty()]
        [String]$Path,
        [ValidateNotNullorEmpty()]
        [String]$Key
    )
    if (!(Test-Path $Path)) {
        return 
    }
    (Get-ItemProperty $Path $Key).$Key
}


Function InitializePackageEnv {
    param(
        [String]$ClangbuilderRoot
    )
    $obj = Get-Content -Path "$ClangbuilderRoot\pkgs\packages.lock.json" |ConvertFrom-Json
    Get-Member -InputObject $obj -MemberType NoteProperty|ForEach-Object {
        $xpath = Find-ExecutablePath -Path "$ClangbuilderRoot\pkgs\$($_.Name)"
        if ($null -ne $xpath) {
            Test-AddPath -Path $xpath
        }
    }
    if (!(Test-ExecuteFile "git")) {
        $gitkey = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\Git_is1"
        $gitkey2 = "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\Git_is1"
        if (Test-Path $gitkey) {
            $gitinstall = Get-RegistryValueEx $gitkey "InstallLocation"
            Test-AddPath "${gitinstall}\bin"
        }
        elseif (Test-Path $gitkey2) {
            $gitinstall = Get-RegistryValueEx $gitkey2 "InstallLocation"
            Test-AddPath "${gitinstall}\bin"
        }
    }
}


Function PMDownload {
    param(
        [String]$Uri, ### URI
        [String]$Path ### save to path
    )
    Write-Host "Download $Uri ..."
    $InternalUA= [Microsoft.PowerShell.Commands.PSUserAgent]::Chrome
    if($Uri.Contains("sourceforge.net")){
        $InternalUA="Clangbuilder/5.0"
    }
	
    try {
        Invoke-WebRequest -Uri $Uri -OutFile $Path -UserAgent $InternalUA -UseBasicParsing
    }
    catch {
        Write-Host -ForegroundColor Red "Download error: $_"
        return $false
    }
    return $true
}

Function Expand-Msi {
    param(
        [String]$Path,
        [String]$DestinationPath ### Full dir of destination path
    )
    $process = Start-Process -FilePath "msiexec" -ArgumentList "/a `"$Path`" /qn TARGETDIR=`"$DestinationPath`""  -PassThru -Wait
    if ($process.ExitCode -ne 0) {
        Write-Host -ForegroundColor Red "Expand-Msi: $Path failed. $($process.ExitCode)"
{'fraction_non_alphanumeric': 0.13687061822817082, 'fraction_numerical': 0.004780114722753346, 'mean_word_length': 2.3656836461126005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051724', 'n_tokens_mistral': 2025, 'n_tokens_neox': 1994, 'n_words': 414}