<commit_before>param($archive_path="")
if( ! $archive_path ) { Write-Error "Argument `"archive_path`" cannot be empty!"; exit 1 }

function Get-ScriptDirectory
{
   $Invocation = (Get-Variable MyInvocation -Scope 1).Value
   Split-Path $Invocation.MyCommand.Path
}

function Power-Balanced
{
   Write-Output "Switching to balanced power scheme"
   & powercfg -setactive 381b4222-f694-41f0-9685-ff5bb260df2e
}

function Power-High
{
   Write-Output "Switching to high power scheme"
   & powercfg -setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c
}

# no line wrapping please!
$host.UI.RawUI.BufferSize = new-object System.Management.Automation.Host.Size(512,50)

# -----------------------------------------------------------------------------
# Following variables could be changed
# -----------------------------------------------------------------------------

$name    = "flibusta"
$retries = 10
$timeout = 90

$mydir   = Get-ScriptDirectory
$wdir    = Join-Path $archive_path ($name + (get-date -format "_yyyyMMdd_HHmmss"))
$adir    = Join-Path $archive_path $name
$glog    = Join-Path $mydir ($name + "_res" + (get-date -format "_yyyyMMdd") + ".log")

# -----------------------------------------------------------------------------
# Main body
# -----------------------------------------------------------------------------

if( $glog ) { Start-Transcript $glog }
Trap { 
    Power-Balanced
	if( $glog ) { Stop-Transcript }; break 
}

Power-High

Write-Output "Downloading $name ..."

& $mydir/libget2 --verbose --library is_$name --retry $retries --timeout $timeout --continue --to $adir --tosql $wdir --config $mydir/libget2.conf 2>&1 | Write-Host

if( $LASTEXITCODE -gt 0 ) { Write-Error "LIBGET error - $LASTEXITCODE !"; Power-Balanced; exit 0 }
if( $LASTEXITCODE -eq 0 ) { Write-Output "No archive updates..."; Power-Balanced; exit 0 }

# Clean old database directories - we have at least one good download
$old_dbs = Join-Path $archive_path "flibusta_*"
Get-ChildItem $old_dbs | Where-Object {$_.PSIsContainer -eq $True} | sort CreationTime -desc | select -Skip 5 | Remove-Item -Force

$new_full_archives = 0
$before_dir = @(Join-path $adir "fb2-*.zip" | dir)

& $mydir/libmerge --verbose --destination $adir 2>&1 | Write-Host

if( $LASTEXITCODE -gt 0 ) { Write-Error "LIBMERGE error - $LASTEXITCODE !"; Power-Balanced; exit 0 }

$after_dir = @(Join-path $adir "fb2-*.zip" | dir)<commit_msg>Fix directory removal<commit_after>param($archive_path="")
if( ! $archive_path ) { Write-Error "Argument `"archive_path`" cannot be empty!"; exit 1 }

function Get-ScriptDirectory
{
   $Invocation = (Get-Variable MyInvocation -Scope 1).Value
   Split-Path $Invocation.MyCommand.Path
}

function Power-Balanced
{
   Write-Output "Switching to balanced power scheme"
   & powercfg -setactive 381b4222-f694-41f0-9685-ff5bb260df2e
}

function Power-High
{
   Write-Output "Switching to high power scheme"
   & powercfg -setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c
}

# no line wrapping please!
$host.UI.RawUI.BufferSize = new-object System.Management.Automation.Host.Size(512,50)

# -----------------------------------------------------------------------------
# Following variables could be changed
# -----------------------------------------------------------------------------

$name    = "flibusta"
$retries = 10
$timeout = 90

$mydir   = Get-ScriptDirectory
$wdir    = Join-Path $archive_path ($name + (get-date -format "_yyyyMMdd_HHmmss"))
$adir    = Join-Path $archive_path $name
$glog    = Join-Path $mydir ($name + "_res" + (get-date -format "_yyyyMMdd") + ".log")

# -----------------------------------------------------------------------------
# Main body
# -----------------------------------------------------------------------------

if( $glog ) { Start-Transcript $glog }
Trap { 
    Power-Balanced
	if( $glog ) { Stop-Transcript }; break 
}

Power-High

Write-Output "Downloading $name ..."

& $mydir/libget2 --verbose --library is_$name --retry $retries --timeout $timeout --continue --to $adir --tosql $wdir --config $mydir/libget2.conf 2>&1 | Write-Host

if( $LASTEXITCODE -gt 0 ) { Write-Error "LIBGET error - $LASTEXITCODE !"; Power-Balanced; exit 0 }
if( $LASTEXITCODE -eq 0 ) { Write-Output "No archive updates..."; Power-Balanced; exit 0 }

# Clean old database directories - we have at least one good download
$old_dbs = Join-Path $archive_path "flibusta_*"
Get-ChildItem $old_dbs | Where-Object {$_.PSIsContainer -eq $True} | sort CreationTime -desc | select -Skip 5 | Remove-Item  -Recurse -Force

$new_full_archives = 0
$before_dir = @(Join-path $adir "fb2-*.zip" | dir)

& $mydir/libmerge --verbose --destination $adir 2>&1 | Write-Host

if( $LASTEXITCODE -gt 0 ) { Write-Error "LIBMERGE error - $LASTEXITCODE !"; Power-Balanced; exit 0 }

$after_dir = @(Join-path $adir "fb2-*.zip" | dir)
{'fraction_non_alphanumeric': 0.25311461794019935, 'fraction_numerical': 0.027408637873754152, 'mean_word_length': 2.2591339648173205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6305216', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1529, 'n_words': 394}