<commit_before>spec: &spec
  x-sub-request-filters:
    - type: default
      name: http
      options:
        allow:
          - pattern: /^https?:\/\//
            forward_headers:
              user-agent: true
  title: The Change Propagation root
  paths:
    /sys/purge:
      x-modules:
        - path: sys/purge.js
          options:
            host: 127.0.0.1
            port: 4321
    /sys/links:
      x-modules:
        - path: sys/dep_updates.js
          options:
            templates:
              mw_api:
                  uri: 'https://{{message.meta.domain}}/w/api.php'
                  headers:
                    host: '{{message.meta.domain}}'
                  body:
                    formatversion: 2
    /sys/queue:
      x-modules:
        - path: sys/kafka.js
          options:
            metadata_broker_list: 127.0.0.1:9092
            dc_name: test_dc
            startup_delay: 0
            consumer:
              # These options should not be copied to puppet config.
              # We're using this config for testing, so need to configure
              # for minimal latency
              fetch.wait.max.ms: "1"
              fetch.min.bytes: "1"
              queue.buffering.max.ms: "1"
            concurrency: 250
            templates:

              summary_definition_rerender: &summary_definition_rerender_spec
                topic: resource_change
                retry_limit: 2
                retry_delay: 500
                retry_on:
                  status:
                    - '5xx'
                cases: # Non wiktionary domains - rerender summary
                  - match:
                      meta:
                        uri: '/^(?<proto>https?):\/\/[^\/]+\/api\/rest_v1\/page\/html\/(?<title>[^/]+)$/'
                      tags:
                        - restbase
                    match_not:
                      - meta:
                          domain: '/wiktionary.org$/'
                      - meta:
                          domain: /\.wikidata\.org$/
                    exec:
                      method: get
                      # Don't encode title since it should be already encoded
                      uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/summary/{{match.meta.uri.title}}'
                      query:
                        redirect: false
                      headers:
                        cache-control: no-cache
                  - match: # Wiktionary domains - rerender definitions
                      meta:
                        # These URIs are coming from RESTBase, so we know that article titles will be normalized
                        # and main namespace articles will not have : (uri-encoded, so %3a or %3A)
                        uri: '/^(?<proto>https?):\/\/[^\/]+\/api\/rest_v1\/page\/html\/(?<title>(?:(?!%3a|%3A|\/).)+)$/'
                        domain: '/^en\.wiktionary\.org$/'
                      tags:
                        - restbase
                    exec:
                      method: get
                      # Don't encode title since it should be already encoded
                      uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/definition/{{match.meta.uri.title}}'
                      query:
                        redirect: false
                      headers:
                        cache-control: no-cache

              summary_definition_rerender_transcludes:
                <<: *summary_definition_rerender_spec
                topic: change-prop.transcludes.resource-change

              mobile_rerender: &mobile_rerender_spec
                topic: resource_change
                retry_limit: 2
                retry_delay: 500
                retry_on:
                  status:
                    - '5xx'
                match:
                  meta:
                    uri: '/^(?<proto>https?):\/\/[^\/]+\/api\/rest_v1\/page\/html\/(?<title>[^/]+)$/'
                  tags:
                    - restbase
                match_not:
                  meta:
                    domain: /\.wikidata\.org$/
                exec:
                  method: get
                  uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/mobile-sections/{{match.meta.uri.title}}'
                  query:
                    redirect: false
                  headers:
                    cache-control: no-cache

              mobile_rerender_transcludes:
                <<: *mobile_rerender_spec
                topic: change-prop.transcludes.resource-change

              purge_varnish: &purge_varnish_spec
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/(?<title>.+)$/'
                  tags:
                    - restbase
                exec:
                  method: post
                  uri: '/sys/purge/'
                  body:
                    - meta:
                        uri: '//{{message.meta.domain}}/api/rest_v1/{{match.meta.uri.title}}'

              purge_varnish_transcludes:
                <<: *purge_varnish_spec
                topic: change-prop.transcludes.resource-change

              # RESTBase update jobs
              mw_purge:
                topic: resource_change
                match:
                  meta:
                    uri: '/^(?<proto>https?):\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - purge
                exec:
                  method: get
                  uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              null_edit:
                topic: resource_change
                ignore:
                  status:
                    - 403 # Ignoring 403 since some of the pages with high number of null_edit events are blacklisted
                    - 412
                match:
                  meta:
                    uri: '/^(?<proto>https?):\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - null_edit
                exec:
                  method: get
                  uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/html/{{match.meta.uri.title}}'<commit_msg>Re-encode page title for mw_purge and null_edit rules<commit_after>spec: &spec
  x-sub-request-filters:
    - type: default
      name: http
      options:
        allow:
          - pattern: /^https?:\/\//
            forward_headers:
              user-agent: true
  title: The Change Propagation root
  paths:
    /sys/purge:
      x-modules:
        - path: sys/purge.js
          options:
            host: 127.0.0.1
            port: 4321
    /sys/links:
      x-modules:
        - path: sys/dep_updates.js
          options:
            templates:
              mw_api:
                  uri: 'https://{{message.meta.domain}}/w/api.php'
                  headers:
                    host: '{{message.meta.domain}}'
                  body:
                    formatversion: 2
    /sys/queue:
      x-modules:
        - path: sys/kafka.js
          options:
            metadata_broker_list: 127.0.0.1:9092
            dc_name: test_dc
            startup_delay: 0
            consumer:
              # These options should not be copied to puppet config.
              # We're using this config for testing, so need to configure
              # for minimal latency
              fetch.wait.max.ms: "1"
              fetch.min.bytes: "1"
              queue.buffering.max.ms: "1"
            concurrency: 250
            templates:

              summary_definition_rerender: &summary_definition_rerender_spec
                topic: resource_change
                retry_limit: 2
                retry_delay: 500
                retry_on:
                  status:
                    - '5xx'
                cases: # Non wiktionary domains - rerender summary
                  - match:
                      meta:
                        uri: '/^(?<proto>https?):\/\/[^\/]+\/api\/rest_v1\/page\/html\/(?<title>[^/]+)$/'
                      tags:
                        - restbase
                    match_not:
                      - meta:
                          domain: '/wiktionary.org$/'
                      - meta:
                          domain: /\.wikidata\.org$/
                    exec:
                      method: get
                      # Don't encode title since it should be already encoded
                      uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/summary/{{match.meta.uri.title}}'
                      query:
                        redirect: false
                      headers:
                        cache-control: no-cache
                  - match: # Wiktionary domains - rerender definitions
                      meta:
                        # These URIs are coming from RESTBase, so we know that article titles will be normalized
                        # and main namespace articles will not have : (uri-encoded, so %3a or %3A)
                        uri: '/^(?<proto>https?):\/\/[^\/]+\/api\/rest_v1\/page\/html\/(?<title>(?:(?!%3a|%3A|\/).)+)$/'
                        domain: '/^en\.wiktionary\.org$/'
                      tags:
                        - restbase
                    exec:
                      method: get
                      # Don't encode title since it should be already encoded
                      uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/definition/{{match.meta.uri.title}}'
                      query:
                        redirect: false
                      headers:
                        cache-control: no-cache

              summary_definition_rerender_transcludes:
                <<: *summary_definition_rerender_spec
                topic: change-prop.transcludes.resource-change

              mobile_rerender: &mobile_rerender_spec
                topic: resource_change
                retry_limit: 2
                retry_delay: 500
                retry_on:
                  status:
                    - '5xx'
                match:
                  meta:
                    uri: '/^(?<proto>https?):\/\/[^\/]+\/api\/rest_v1\/page\/html\/(?<title>[^/]+)$/'
                  tags:
                    - restbase
                match_not:
                  meta:
                    domain: /\.wikidata\.org$/
                exec:
                  method: get
                  uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/mobile-sections/{{match.meta.uri.title}}'
                  query:
                    redirect: false
                  headers:
                    cache-control: no-cache

              mobile_rerender_transcludes:
                <<: *mobile_rerender_spec
                topic: change-prop.transcludes.resource-change

              purge_varnish: &purge_varnish_spec
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/(?<title>.+)$/'
                  tags:
                    - restbase
                exec:
                  method: post
                  uri: '/sys/purge/'
                  body:
                    - meta:
                        uri: '//{{message.meta.domain}}/api/rest_v1/{{match.meta.uri.title}}'

              purge_varnish_transcludes:
                <<: *purge_varnish_spec
                topic: change-prop.transcludes.resource-change

              # RESTBase update jobs
              mw_purge:
                topic: resource_change
                match:
                  meta:
                    uri: '/^(?<proto>https?):\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - purge
                exec:
                  method: get
                  # This even comes directly from MediaWiki, so title is encoded in MW-specific way.
                  # Re-encode the title in standard `encodeURIComponent` encoding.
                  uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/html/{decode(match.meta.uri.title)}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              null_edit:
                topic: resource_change
                ignore:
                  status:
                    - 403 # Ignoring 403 since some of the pages with high number of null_edit events are blacklisted
                    - 412
                match:
                  meta:
                    uri: '/^(?<proto>https?):\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - null_edit
                exec:
                  method: get
                  # This even comes directly from MediaWiki, so title is encoded in MW-specific way.
                  # Re-encode the title in standard `encodeURIComponent` encoding.
                  uri: '{{match.meta.uri.proto}}://{{message.meta.domain}}/api/rest_v1/page/html/{decode(match.meta.uri.title)}'
{'fraction_non_alphanumeric': 0.11349147516679021, 'fraction_numerical': 0.009043736100815419, 'mean_word_length': 0.9983706117612205, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3803109', 'n_tokens_mistral': 3732, 'n_tokens_neox': 3323, 'n_words': 680}