<commit_before>Param(

    $path

)

if (-not $path) {$path = '.'}

(storj list-buckets | sls "id: (.*), Name: (.*?), Storage:").matches | %{

    $bucketName = $_.Groups[2].value;

    $bucketID = $_.Groups[1].Value;

    $bucketFiles = (storj list-files $bucketID | sls "Name: (.*), type:.* id: (.*)").matches;

    Get-ChildItem (Join-Path $path $bucketName) -Recurse | ? length -gt 1000 | %{

        $isFnd = $false;

        ForEach ($bFile in $bucketFiles) {

            Write-Information $bFile.Groups[1].Value "== $_ ?";

            if ($_.ToString() -eq $bFile.Groups[1].Value.ToString()) {

                Write-Information $_ "found";

                $isFnd = $true;

                break;

            }

        }

        if (-not $isFnd) {

            Write-Information "storj upload-file $bucketID" $_.FullName;

            storj upload-file $bucketID $_.FullName

        }

    }

}

<commit_msg>+ exit on error occurred<commit_after>Param(

    $path

)

if (-not $path) {$path = '.'}

(storj list-buckets | sls "id: (.*), Name: (.*?), Storage:").matches | %{

    $bucketName = $_.Groups[2].value;

    $bucketID = $_.Groups[1].Value;

    $bucketFiles = (storj list-files $bucketID | sls "Name: (.*), type:.* id: (.*)").matches;

    Get-ChildItem (Join-Path $path $bucketName) -Recurse | ? length -gt 1000 | %{

        $isFnd = $false;

        ForEach ($bFile in $bucketFiles) {

            Write-Information $bFile.Groups[1].Value "== $_ ?";

            if ($_.ToString() -eq $bFile.Groups[1].Value.ToString()) {

                Write-Information $_ "found";

                $isFnd = $true;

                break;

            }

        }

        if (-not $isFnd) {

            Write-Information "storj upload-file $bucketID" $_.FullName;

            storj upload-file $bucketID $_.FullName | % {

                [System.Console]::WriteLine($_);

                if ($_.contains("error")) {Exit}

            }

        }

    }

}


{'fraction_non_alphanumeric': 0.5827116027784924, 'fraction_numerical': 0.004116285052739903, 'mean_word_length': 4.491525423728813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '442225', 'n_tokens_mistral': 3843, 'n_tokens_neox': 3842, 'n_words': 132}