<commit_before>{ lib, makeEffect, google-cloud-sdk }:

{ bucket, object, name, file, contentType, serviceAccountKey }:

assert lib.asserts.assertMsg (builtins.isString serviceAccountKey)
  "`serviceAccountKey` must contain the JSON contents of a service-account key";

let

  uri = "gs://${bucket}/${lib.removePrefix "/" object}";

in makeEffect {
  inherit file uri contentType serviceAccountKey;

  name = "push-${name}";

  inputs = [ google-cloud-sdk ];

  dontUnpack = true;

  effectScript = ''
    export HOME="."

    gcloud auth activate-service-account --key-file=- <<< $serviceAccountKey

    stat_uri() {
      gsutil stat $uri
    }

    if ! stat_uri; then
       header "copying $file to $uri"

       gsutil -h "Content-Type:$contentType" cp $file $uri

       if ! stat_uri; then
         echo "failed pushing $file to $url" >&2
         exit 1 
       fi
    fi 
  '';
}<commit_msg>build: force google-cloud-sdk to use python3<commit_after>{ lib, makeEffect, python3, google-cloud-sdk }:

{ bucket, object, name, file, contentType, serviceAccountKey }:

assert lib.asserts.assertMsg (builtins.isString serviceAccountKey)
  "`serviceAccountKey` must contain the JSON contents of a service-account key";

let

  uri = "gs://${bucket}/${lib.removePrefix "/" object}";

in makeEffect {
  inherit file uri contentType serviceAccountKey;

  name = "push-${name}";

  inputs = [ (google-cloud-sdk.override { python = python3; }) ];

  dontUnpack = true;

  effectScript = ''
    export HOME="."

    gcloud auth activate-service-account --key-file=- <<< $serviceAccountKey

    stat_uri() {
      gsutil stat $uri
    }

    if ! stat_uri; then
       header "copying $file to $uri"

       gsutil -h "Content-Type:$contentType" cp $file $uri

       if ! stat_uri; then
         echo "failed pushing $file to $url" >&2
         exit 1 
       fi
    fi 
  '';
}
{'fraction_non_alphanumeric': 0.12760277629471437, 'fraction_numerical': 0.0037373198077949813, 'mean_word_length': 2.9287211740041927, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1984153', 'n_tokens_mistral': 615, 'n_tokens_neox': 581, 'n_words': 170}