<commit_before>"apiVersion": "batch/v1beta1"
"kind": "CronJob"
"metadata":
  "name": "mnist-v2-8-example"
"spec":
  "concurrencyPolicy": "Forbid"
  "jobTemplate":
    "spec":
      "backoffLimit": 0
      "template":
        "metadata":
          "annotations":
            "tf-version.cloud-tpus.google.com": "nightly-2.x"
        "spec":
          "activeDeadlineSeconds": 3600
          "containers":
          - "args":
            - "python3"
            - "official/vision/image_classification/mnist_main.py"
            - "--data_dir=gs://xl-ml-test-us-central1/data/mnist"
            - "--distribution_strategy=tpu"
            - "--batch_size=1024"
            - "--train_epochs=1"
            - "--epochs_between_evals=1"
            - "--model_dir=$(MODEL_DIR)"
            "env":
            - "name": "POD_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.name"
            - "name": "POD_UID"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.uid"
            - "name": "POD_NAMESPACE"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.namespace"
            - "name": "JOB_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.labels['job-name']"
            - "name": "MODEL_DIR"
              "value": "gs://xl-ml-test-us-central1/k8s/mnist/functional/v2-8/$(JOB_NAME)"
            - "name": "TEST_NAME"
              "value": "tf-mnist-v2-8"
            - "name": "METRIC_COLLECTION_CONFIG"
              "value": |
                {
                    "write_to_bigquery": "True",
                    "bigquery_table_name": "xl_ml_metrics_table",
                    "bigquery_dataset_name": "xl_ml_metrics_dataset",
                    "tags_to_ignore": ["LearningRate"],
                    "default_aggregation_strategies": ["final"],
                    "metric_to_aggregation_strategies": {
                        "epoch_loss": ["final", "min"]
                    },
                   "time_to_accuracy": {
                       "accuracy_threshold": 99.0,
                       "accuracy_tag": "epoch_sparse_categorical_accuracy"
                   }
               }
            - "name": "REGRESSION_TEST_CONFIG"
              "value": |
                {
                     "write_metrics_to_stackdriver": "True",
                     "write_alerts_to_stackdriver": "True",
                     "min_num_datapoints_before_alerting": 1,
                     "metrics_to_ignore": ["loss"],
                     "notification_channel_display_names": ["default_notification_channel"],
                     "base_threshold_expression": "v_mean + (v_stddev * 6.0)",
                     "base_comparison": "COMPARISON_GT",
                     "threshold_expression_overrides": {
                          "epoch_sparse_categorical_accuracy_final": "v_mean - (v_stddev * 3.0)"
                     },
                     "comparison_overrides": {
                         "epoch_sparse_categorical_accuracy_final": "COMPARISON_LT"
                     }
                }
            "image": "gcr.io/xl-ml-test/model-garden:nightly"
            "imagePullPolicy": "Always"
            "name": "mnist-functional-v2-8"
            "resources":
              "limits":
                "cloud-tpus.google.com/preemptible-v2": 8
          "restartPolicy": "Never"
  "schedule": "0 */24 * * *"<commit_msg>fix indent in tf mnist yaml<commit_after>"apiVersion": "batch/v1beta1"
"kind": "CronJob"
"metadata":
  "name": "mnist-v2-8-example"
"spec":
  "concurrencyPolicy": "Forbid"
  "jobTemplate":
    "spec":
      "backoffLimit": 0
      "template":
        "metadata":
          "annotations":
            "tf-version.cloud-tpus.google.com": "nightly-2.x"
        "spec":
          "activeDeadlineSeconds": 3600
          "containers":
          - "args":
            - "python3"
            - "official/vision/image_classification/mnist_main.py"
            - "--data_dir=gs://xl-ml-test-us-central1/data/mnist"
            - "--distribution_strategy=tpu"
            - "--batch_size=1024"
            - "--train_epochs=1"
            - "--epochs_between_evals=1"
            - "--model_dir=$(MODEL_DIR)"
            "env":
            - "name": "POD_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.name"
            - "name": "POD_UID"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.uid"
            - "name": "POD_NAMESPACE"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.namespace"
            - "name": "JOB_NAME"
              "valueFrom":
                "fieldRef":
                  "fieldPath": "metadata.labels['job-name']"
            - "name": "MODEL_DIR"
              "value": "gs://xl-ml-test-us-central1/k8s/mnist/functional/v2-8/$(JOB_NAME)"
            - "name": "TEST_NAME"
              "value": "tf-mnist-v2-8"
            - "name": "METRIC_COLLECTION_CONFIG"
              "value": |
                {
                    "write_to_bigquery": "True",
                    "bigquery_dataset_name": "xl_ml_metrics_dataset",
                    "bigquery_table_name": "xl_ml_metrics_table",
                    "tags_to_ignore": ["LearningRate"],
                    "default_aggregation_strategies": ["final"],
                    "metric_to_aggregation_strategies": {
                        "epoch_loss": ["final", "min"]
                    },
                    "time_to_accuracy": {
                       "accuracy_threshold": 99.0,
                       "accuracy_tag": "epoch_sparse_categorical_accuracy"
                    }
                 }
            - "name": "REGRESSION_TEST_CONFIG"
              "value": |
                {
                     "write_metrics_to_stackdriver": "True",
                     "write_alerts_to_stackdriver": "True",
                     "min_num_datapoints_before_alerting": 1,
                     "metrics_to_ignore": ["loss"],
                     "notification_channel_display_names": ["default_notification_channel"],
                     "base_threshold_expression": "v_mean + (v_stddev * 6.0)",
                     "base_comparison": "COMPARISON_GT",
                     "threshold_expression_overrides": {
                          "epoch_sparse_categorical_accuracy_final": "v_mean - (v_stddev * 3.0)"
                     },
                     "comparison_overrides": {
                         "epoch_sparse_categorical_accuracy_final": "COMPARISON_LT"
                     }
                }
            "image": "gcr.io/xl-ml-test/model-garden:nightly"
            "imagePullPolicy": "Always"
            "name": "mnist-functional-v2-8"
            "resources":
              "limits":
                "cloud-tpus.google.com/preemptible-v2": 8
          "restartPolicy": "Never"
  "schedule": "0 */24 * * *"

{'fraction_non_alphanumeric': 0.13386161030140087, 'fraction_numerical': 0.011037215225696902, 'mean_word_length': 1.375798319327731, 'pattern_counts': {'":': 142, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2325597', 'n_tokens_mistral': 2152, 'n_tokens_neox': 2038, 'n_words': 263}