<commit_before># URL Configuration
$packageDir = $args[0] # Directory to save packages to
$release = $args[1] # Package release to download from

Write-Output "Scanning packages..."

# Enumerate the list of packages
foreach ($package in $args[2..$args.Length])
{
    $packagePath = $packageDir + $package

    # Check if we've already downloaded the package
    if (Test-Path $packagePath)
    {
        Write-Output ("Package {0} up to date." -f $package)
        continue
    }

    $temp = [System.IO.Path]::GetTempFileName() + ".zip"

    # Download the package
    Write-Output ("Downloading package {0}..." -f $package)
    $url = "https://github.com/willcassella/SinGE/releases/download/{0}/{1}.zip" -f $release, $package
    Invoke-WebRequest $url -outfile $temp

    # Extract the package
    Write-Output ("Extracting package {0}..." -f $package)
    Expand-Archive $temp -dest $packagePath
    Remove-Item $temp
}<commit_msg>Made package downloads WAY faster<commit_after># URL Configuration
$packageDir = $args[0] # Directory to save packages to
$release = $args[1] # Package release to download from

Write-Output "Scanning packages..."

# Enumerate the list of packages
foreach ($package in $args[2..$args.Length])
{
    $packagePath = $packageDir + $package

    # Check if we've already downloaded the package
    if (Test-Path $packagePath)
    {
        Write-Output ("Package {0} is up to date." -f $package)
        continue
    }

    $temp = [System.IO.Path]::GetTempFileName() + ".zip"

    # Download the package
    Write-Output ("Downloading package {0}..." -f $package)
    $url = "https://github.com/willcassella/SinGE/releases/download/{0}/{1}.zip" -f $release, $package
    $web = New-Object net.webclient
    $web.DownloadFile($url, $temp)

    # Extract the package
    Write-Output ("Extracting package {0}..." -f $package)
    Expand-Archive $temp -dest $packagePath
    Remove-Item $temp
}
{'fraction_non_alphanumeric': 0.1398128898128898, 'fraction_numerical': 0.008316008316008316, 'mean_word_length': 3.230769230769231, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4470619', 'n_tokens_mistral': 610, 'n_tokens_neox': 578, 'n_words': 180}