<commit_before><commit_msg>add dashboard for VPC flow logs<commit_after>{
  "displayName": "VPC Flow Logs",
  "mosaicLayout": {
    "columns": 12,
    "tiles": [
      {
        "height": 4,
        "widget": {
          "title": "Log bytes ingested for gce_subnetwork by label.resource_type [SUM]",
          "xyChart": {
            "chartOptions": {
              "mode": "COLOR"
            },
            "dataSets": [
              {
                "minAlignmentPeriod": "60s",
                "plotType": "LINE",
                "timeSeriesQuery": {
                  "timeSeriesFilter": {
                    "aggregation": {
                      "perSeriesAligner": "ALIGN_RATE"
                    },
                    "filter": "metric.type=\"logging.googleapis.com/billing/bytes_ingested\" resource.type=\"global\" metric.label.\"resource_type\"=\"gce_subnetwork\"",
                    "secondaryAggregation": {
                      "perSeriesAligner": "ALIGN_MEAN"
                    }
                  }
                }
              }
            ],
            "timeshiftDuration": "0s",
            "yAxis": {
              "label": "y1Axis",
              "scale": "LINEAR"
            }
          }
        },
        "width": 6,
        "yPos": 2
      },
      {
        "height": 4,
        "widget": {
          "title": "gce_subnetwork log entries count [SUM]",
          "xyChart": {
            "chartOptions": {
              "mode": "COLOR"
            },
            "dataSets": [
              {
                "minAlignmentPeriod": "60s",
                "plotType": "LINE",
                "timeSeriesQuery": {
                  "timeSeriesFilter": {
                    "aggregation": {
                      "crossSeriesReducer": "REDUCE_SUM",
                      "perSeriesAligner": "ALIGN_SUM"
                    },
                    "filter": "metric.type=\"logging.googleapis.com/log_entry_count\" resource.type=\"gce_subnetwork\""
                  }
                }
              }
            ],
            "timeshiftDuration": "0s",
            "yAxis": {
              "label": "y1Axis",
              "scale": "LINEAR"
            }
          }
        },
        "width": 6,
        "xPos": 6,
        "yPos": 2
      },
      {
        "height": 4,
        "widget": {
          "title": "Log bytes ingested by label.resource_type [SUM]",
          "xyChart": {
            "chartOptions": {
              "mode": "COLOR"
            },
            "dataSets": [
              {
                "minAlignmentPeriod": "60s",
                "plotType": "LINE",
                "timeSeriesQuery": {
                  "timeSeriesFilter": {
                    "aggregation": {
                      "perSeriesAligner": "ALIGN_RATE"
                    },
                    "filter": "metric.type=\"logging.googleapis.com/billing/bytes_ingested\" resource.type=\"global\"",
                    "secondaryAggregation": {
                      "perSeriesAligner": "ALIGN_MEAN"
                    }
                  }
                }
              }
            ],
            "timeshiftDuration": "0s",
            "yAxis": {
              "label": "y1Axis",
              "scale": "LINEAR"
            }
          }
        },
        "width": 6,
        "yPos": 6
      },
      {
        "height": 4,
        "widget": {
          "title": "Predicted Max VPC Flow Logs Count [SUM]",
          "xyChart": {
            "chartOptions": {
              "mode": "COLOR"
            },
            "dataSets": [
              {
                "minAlignmentPeriod": "60s",
                "plotType": "LINE",
                "timeSeriesQuery": {
                  "timeSeriesFilter": {
                    "aggregation": {
                      "crossSeriesReducer": "REDUCE_SUM",
                      "groupByFields": [
                        "resource.label.\"project_id\""
                      ],
                      "perSeriesAligner": "ALIGN_SUM"
                    },
                    "filter": "metric.type=\"networking.googleapis.com/vpc_flow/predicted_max_vpc_flow_logs_count\" resource.type=\"gce_subnetwork\""
                  }
                }
              }
            ],
            "timeshiftDuration": "0s",
            "yAxis": {
              "label": "y1Axis",
              "scale": "LINEAR"
            }
          }
        },
        "width": 6,
        "xPos": 6,
        "yPos": 12
      },
      {
        "height": 4,
        "widget": {
          "title": "Log entries count [SUM]",
          "xyChart": {
            "chartOptions": {
              "mode": "COLOR"
            },
            "dataSets": [
              {
                "minAlignmentPeriod": "60s",
                "plotType": "LINE",
                "timeSeriesQuery": {
                  "timeSeriesFilter": {
                    "aggregation": {
                      "crossSeriesReducer": "REDUCE_SUM",
                      "perSeriesAligner": "ALIGN_SUM"
                    },
                    "filter": "metric.type=\"logging.googleapis.com/log_entry_count\" resource.type=\"global\""
                  }
                }
              }
            ],
            "timeshiftDuration": "0s",
            "yAxis": {
              "label": "y1Axis",
              "scale": "LINEAR"
            }
          }
        },
        "width": 6,
        "xPos": 6,
        "yPos": 6
      },
      {
        "height": 2,
        "widget": {
          "text": {
            "content": "These charts represent the volume of logs ingested for VPC Flow logs. The chart below provides the total volume of logs ingested for reference.",
            "format": "RAW"
          },
          "title": "Ingested log volume"
        },
        "width": 6
      },
      {
        "height": 2,
        "widget": {
          "text": {
            "content": "These 2 charts represent the count of log entries for VPC Flow logs. The chart below provides the total count of log entries for reference.",
            "format": "RAW"
          },
          "title": "Log counts"
        },
        "width": 6,
        "xPos": 6
      },
      {
        "height": 2,
        "widget": {
          "text": {
            "content": "The metric predicted_max_vpc_flow_logs_count is used for the two charts below. They can be used to help estimate VPC Flow log counts and volume. For more details see the metric [vpc_flow/predicted_max_vpc_flow_logs_count ](https://cloud.google.com/monitoring/api/metrics_gcp#gcp-networking).",
            "format": "MARKDOWN"
          },
          "title": "VPC Flow logs predicted count"
        },
        "width": 12,
        "yPos": 10
      },
      {
        "height": 4,
        "widget": {
          "title": "VCP Flow Logs Volume",
          "xyChart": {
            "chartOptions": {
              "mode": "COLOR"
            },
            "dataSets": [
              {
                "plotType": "LINE",
                "timeSeriesQuery": {
                  "timeSeriesQueryLanguage": "fetch gce_subnetwork\n| metric 'networking.googleapis.com/vpc_flow/predicted_max_vpc_flow_logs_count'\n| align mean_aligner(1w)\n| every 1w\n| group_by [],\n    [value_predicted_max_vpc_flow_logs_count_aggregate:\n       aggregate(value.predicted_max_vpc_flow_logs_count)]\n| value\n    [value_predicted_max_vpc_flow_logs_count_aggregate_mul:\n       mul(value_predicted_max_vpc_flow_logs_count_aggregate, 1080)]\n| value\n    [value_predicted_max_vpc_flow_logs_count_aggregate_mul_cast_units:\n       cast_units(value_predicted_max_vpc_flow_logs_count_aggregate_mul, 'By')]\n| value\n    [value_predicted_max_vpc_flow_logs_count_aggregate_mul_cast_units_mul:\n       mul(value_predicted_max_vpc_flow_logs_count_aggregate_mul_cast_units,\n         60 * 24 * 30)]\n| value\n    [value_predicted_max_vpc_flow_logs_count_aggregate_mul_cast_units_mul_mul:\n       mul(\n         value_predicted_max_vpc_flow_logs_count_aggregate_mul_cast_units_mul,\n         1.0)]"
                }
              }
            ],
            "timeshiftDuration": "0s",
            "yAxis": {
              "label": "y1Axis",
              "scale": "LINEAR"
            }
          }
        },
        "width": 6,
        "yPos": 12
      }
    ]
  }
}
{'fraction_non_alphanumeric': 0.12627658296287397, 'fraction_numerical': 0.008650726901357684, 'mean_word_length': 1.232233842853312, 'pattern_counts': {'":': 154, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '670779', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2296, 'n_words': 408}