<commit_before>pipeline:
  threads: 20
  processors:
  - resource: scrape_third_party_downloads
  - catch:
      - bloblang: root = error()
  - bloblang: result = content().string()

resources:
  processors:
    scrape_third_party_downloads:
      try:
      # Grab docker dl count
      - bloblang: root = ""
      - http:
          url: https://hub.docker.com/v2/repositories/jeffail/benthos/
          verb: GET
          retries: 0
      - bloblang: |
          type = "gauge"
          source = "docker"
          dist = "docker"
          download_count = pull_count
      - resource: set_metrics

      # Grab github latest release dl count
      - bloblang: root = ""
      - http:
          url: https://api.github.com/repos/Jeffail/benthos/releases/latest
          verb: GET
          retries: 0
      - bloblang: |
          root = assets.map_each(match {
            "type":"gauge",
            "source":"github",
            "dist": name.re_replace("^benthos-?((lambda_)|_)[0-9\\.]+_([^\\.]+).*", "$2$3"),
            "download_count": download_count
          } {
            dist != "checksums" => this
            _ => deleted()
          })
      - unarchive:
          format: json_array
      - resource: set_metrics

    set_metrics:
      try:
        - switch:
           - condition:
               bloblang: type == "gauge"
             processors:
               - resource: metric.gauge
           - processors:
               - resource: metric.counter

    metric.gauge:
      metric:
        labels:
          dist: ${!json("dist")}
          source: ${!json("source")}
        path: BenthosDownloadGauge
        type: gauge
        value: ${!json("download_count")}

    metric.counter:
      metric:
        labels:
          version: ${!json("version")}
          dist: ${!json("dist")}
          source: ${!json("source")}
        path: BenthosDownload
        type: counter

metrics:
  rename:
    by_regexp:
    - pattern: '^resource\.processor\.metric\.(gauge|counter)\.(.*)$'
      value: "$2"
    child:
      whitelist:
        paths:
          - BenthosDownloadGauge
          - BenthosDownload
        child:
          prometheus:
            prefix: benthos<commit_msg>Update download scraper example with workflow<commit_after>pipeline:
  threads: 20
  processors:
    - bloblang: 'root = {}'
    - workflow:
        meta_path: results
        branches:
          dockerhub:
            request_map: 'root = ""'
            processors:
              - try:
                # Grab docker dl count
                - http:
                    url: https://hub.docker.com/v2/repositories/jeffail/benthos/
                    verb: GET
                    retries: 0
                - bloblang: |
                    type = "gauge"
                    source = "docker"
                    dist = "docker"
                    download_count = pull_count
                - resource: set_metrics

          github:
            request_map: 'root = ""'
            processors:
              - try:
                # Grab github latest release dl count
                - http:
                    url: https://api.github.com/repos/Jeffail/benthos/releases/latest
                    verb: GET
                    retries: 0
                - bloblang: |
                    root = assets.map_each(match {
                      "type":"gauge",
                      "source":"github",
                      "dist": name.re_replace("^benthos-?((lambda_)|_)[0-9\\.]+_([^\\.]+).*", "$2$3"),
                      "download_count": download_count
                    } {
                      dist != "checksums" => this
                      _ => deleted()
                    })
                - unarchive:
                    format: json_array
                - resource: set_metrics
                - bloblang: 'root = if batch_index() != 0 { deleted() }'

          homebrew:
            request_map: 'root = ""'
            processors:
              - try:
                - http:
                    url: https://formulae.brew.sh/api/formula/benthos.json
                    verb: GET
                    retries: 0
                - bloblang: |
                    type = "gauge"
                    source = "homebrew"
                    dist = "brew"
                    download_count = analytics.install.30d.benthos
                - resource: set_metrics

resources:
  processors:
    set_metrics:
      try:
        - switch:
           - condition:
               bloblang: type == "gauge"
             processors:
               - resource: metric.gauge
           - processors:
               - resource: metric.counter

    metric.gauge:
      metric:
        labels:
          dist: ${!json("dist")}
          source: ${!json("source")}
        path: BenthosDownloadGauge
        type: gauge
        value: ${!json("download_count")}

    metric.counter:
      metric:
        labels:
          version: ${!json("version")}
          dist: ${!json("dist")}
          source: ${!json("source")}
        path: BenthosDownload
        type: counter

metrics:
  cloudwatch:
    namespace: BenthosAnalyticsStaging
    flush_period: 500ms
    region: eu-west-1
    path_mapping: |
      let name = this.re_replace("^resource\\.processor\\.metric\\.(gauge|counter)\\.(.*)$", "$2")
      root = if [
        "BenthosDownloadGauge",
        "BenthosDownload"
      ].contains($name) { $name } else { deleted() }
{'fraction_non_alphanumeric': 0.11991199119911991, 'fraction_numerical': 0.005133846718005134, 'mean_word_length': 1.282426778242678, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3822931', 'n_tokens_mistral': 1577, 'n_tokens_neox': 1471, 'n_words': 299}