<commit_before>$MyPath = [IO.Path]::GetDirectoryName($MyInvocation.MyCommand.Definition)

$BasePath = Resolve-Path "$MyPath\.."
$BasePath = "$BasePath\tools"
if (!(Test-Path $BasePath)) {
	mkdir $BasePath | Out-Null
}

$7zipUrl = "https://sourceforge.net/projects/sevenzip/files/7-Zip/9.20/7za920.zip/download"
$7zipDir = "$BasePath\7z"

$ffmpegUrl = "http://ffmpeg.zeranoe.com/builds/win32/static/ffmpeg-latest-win32-static.7z"
$ffmpegDir = "$BasePath\FFmpeg"

$wavevizUrl = "https://github.com/mastersign/waveviz/releases/download/v1.1.0/WaveViz.exe"
$wavevizDir = "$BasePath\WaveViz"


function unzip($filePath, $targetDir) {
    if (!(Test-Path $targetDir)) { mkdir $targetDir | Out-Null }
    $shellApp = New-Object -com shell.application
    $zipFile = $shellApp.namespace($filePath)
    $dest = $shellApp.namespace($targetDir)
    $dest.Copyhere($zipFile.items(), 0x10)
}

function un7zip($filePath, $targetDir) {
    & "$script:7zipDir\7za.exe" x "-o$targetDir" $filePath
}

function download($url, $targetFile) {
    if (Test-Path $targetFile) {
        Remove-Item $targetFile
    }
    $wc = New-Object System.Net.WebClient
    $wc.DownloadFile($url, $targetFile)
}

## Download 7zip

$tmp = "$BasePath\download.zip"
download $7zipUrl $tmp
unzip $tmp $7zipDir
Remove-Item $tmp

## Download FFmpeg

$tmp = "$BasePath\download.7z"
download $ffmpegUrl $tmp
un7zip $tmp $BasePath
Remove-Item $tmp
if (Test-Path $ffmpegDir) { Remove-Item $ffmpegDir -Recurse -Force }
Get-ChildItem "$BasePath\ffmpeg-*-win32-static" -Directory `
    | % { Move-Item $_ $ffmpegDir }

## Download WaveViz

if (!(Test-Path $wavevizDir)) { mkdir $wavevizDir | Out-Null }
download $wavevizUrl "$wavevizDir\WaveViz.exe"<commit_msg>do not download already existing tools<commit_after>$MyPath = [IO.Path]::GetDirectoryName($MyInvocation.MyCommand.Definition)

$BasePath = Resolve-Path "$MyPath\.."
$BasePath = "$BasePath\tools"
if (!(Test-Path $BasePath)) {
	mkdir $BasePath | Out-Null
}

$7zipUrl = "https://sourceforge.net/projects/sevenzip/files/7-Zip/9.20/7za920.zip/download"
$7zipDir = "$BasePath\7z"

$ffmpegUrl = "http://ffmpeg.zeranoe.com/builds/win32/static/ffmpeg-latest-win32-static.7z"
$ffmpegDir = "$BasePath\FFmpeg"

$wavevizUrl = "https://github.com/mastersign/waveviz/releases/download/v1.1.0/WaveViz.exe"
$wavevizDir = "$BasePath\WaveViz"


function unzip($filePath, $targetDir) {
    if (!(Test-Path $targetDir)) { mkdir $targetDir | Out-Null }
    $shellApp = New-Object -com shell.application
    $zipFile = $shellApp.namespace($filePath)
    $dest = $shellApp.namespace($targetDir)
    $dest.Copyhere($zipFile.items(), 0x10)
}

function un7zip($filePath, $targetDir) {
    & "$script:7zipDir\7za.exe" x "-o$targetDir" $filePath
}

function download($url, $targetFile) {
    if (Test-Path $targetFile) {
        Remove-Item $targetFile
    }
    $wc = New-Object System.Net.WebClient
    $wc.DownloadFile($url, $targetFile)
}

## Download 7zip

if (!(Test-Path "$7zipDir\7za.exe")) {
	$tmp = "$BasePath\download.zip"
	download $7zipUrl $tmp
	unzip $tmp $7zipDir
	Remove-Item $tmp
}

## Download FFmpeg

if (!(Test-Path "$ffmpegDir\bin\ffmpeg.exe")) {
	$tmp = "$BasePath\download.7z"
	download $ffmpegUrl $tmp
	un7zip $tmp $BasePath
	Remove-Item $tmp
	if (Test-Path $ffmpegDir) { Remove-Item $ffmpegDir -Recurse -Force }
	Get-ChildItem "$BasePath\ffmpeg-*-win32-static" -Directory `
		| % { Move-Item $_ $ffmpegDir }
}

## Download WaveViz

if (!(Test-Path "$wavevizDir\WaveViz.exe")) {
	if (!(Test-Path $wavevizDir)) { mkdir $wavevizDir | Out-Null }
	download $wavevizUrl "$wavevizDir\WaveViz.exe"
}
{'fraction_non_alphanumeric': 0.16833333333333333, 'fraction_numerical': 0.018333333333333333, 'mean_word_length': 4.447806354009077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2318693', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1382, 'n_words': 235}